"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
internxt: fix sync with --backup-dir or --suffix deleting the backed up file
Nick Craig-Woodf068abd607f8
diff --git a/backend/internxt/internxt.go b/backend/internxt/internxt.goindex 89f144c64..598384439 100644--- a/backend/internxt/internxt.go+++ b/backend/internxt/internxt.go@@ -592,6 +592,73 @@ func existenceCheck(name, ext string) files.FileExistenceCheck {}}+// staleWindow is how long the API may carry on returning an item from+// where it was before this process moved or deleted it.+const staleWindow = time.Minute++// location is where this process last put an item. An empty parentUUID+// means the item was deleted.+type location struct {+ parentUUID string+ name string // encoded name, including any extension+ expires time.Time+}++// recentChanges holds the files and folders this process has moved or+// deleted recently, keyed by UUID.+//+// The API serves lookups and listings from read replicas which lag behind+// writes, so an item can be returned from its old location for a while after+// it was moved or deleted. Acting on such a stale entry can be destructive,+// eg overwriting it would delete the file from where it was moved to.+type recentChanges struct {+ mu sync.Mutex+ items map[string]location+ lastPrune time.Time+}++// recent is shared by all the Fs in the process as UUIDs are global.+var recent = recentChanges{items: map[string]location{}}++// set records the new location of uuid, pruning expired entries.+func (r *recentChanges) set(uuid string, loc location) {+ r.mu.Lock()+ defer r.mu.Unlock()+ now := time.Now()+ if now.Sub(r.lastPrune) > staleWindow {+ for k, v := range r.items {+ if now.After(v.expires) {+ delete(r.items, k)+ }+ }+ r.lastPrune = now+ }+ loc.expires = now.Add(staleWindow)+ r.items[uuid] = loc+}++// moved records that uuid was moved to name in parentUUID.+func (r *recentChanges) moved(uuid, parentUUID, name string) {+ r.set(uuid, location{parentUUID: parentUUID, name: name})+}++// deleted records that uuid was deleted.+func (r *recentChanges) deleted(uuid string) {+ r.set(uuid, location{})+}++// stale reports whether an API result saying uuid is called name in+// parentUUID is out of date because of a change made by this process.+func (r *recentChanges) stale(uuid, parentUUID, name string) bool {+ r.mu.Lock()+ defer r.mu.Unlock()+ loc, ok := r.items[uuid]+ if !ok || time.Now().After(loc.expires) {+ return false+ }+ return loc.parentUUID != parentUUID || loc.name != name+}+// findFile looks up a single file by name within directoryID. Returns// (nil, nil) when the file does not exist; surfaces transport/API errors.func (f *Fs) findFile(ctx context.Context, leaf, directoryID string) (*folders.File, error) {@@ -623,6 +690,10 @@ func (f *Fs) findFile(ctx context.Context, leaf, directoryID string) (*folders.Fif !result.FileExists() || result.UUID == "" {continue}+ if recent.stale(result.UUID, directoryID, joinNameExt(result.PlainName, result.Type)) {+ fs.Debugf(f, "Ignoring stale lookup of %q (UUID: %s) which was moved or deleted", leaf, result.UUID)+ continue+ }if joinNameExt(result.PlainName, result.Type) == encodedName {uuid = result.UUIDbreak@@ -699,6 +770,9 @@ func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) {if len(e.Type) > 0 {remote += "." + e.Type}+ if recent.stale(e.UUID, dirID, remote) {+ continue+ }remote = path.Join(dir, f.opt.Encoding.ToStandardName(remote))out = append(out, newObjectWithFile(f, remote, &e))}@@ -905,6 +979,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,if err != nil {return nil, err}+ recent.moved(srcObj.uuid, directoryID, joinNameExt(newName, newType))dstObj := &Object{f: f,@@ -1124,6 +1199,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, opbackupName, backupType, backupUUID, err)// Don't fail the upload just because backup deletion failed} else {+ recent.deleted(backupUUID)fs.Debugf(o.f, "Successfully deleted backup file")}}@@ -1229,8 +1305,13 @@ func (o *Object) restoreBackupFile(ctx context.Context, backupUUID, origName, or// Remove deletes a filefunc (o *Object) Remove(ctx context.Context) error {- return o.f.pacer.Call(func() (bool, error) {+ err := o.f.pacer.Call(func() (bool, error) {err := files.DeleteFile(ctx, o.f.cfg, o.uuid)return o.f.shouldRetry(ctx, err)})+ if err != nil {+ return err+ }+ recent.deleted(o.uuid)+ return nil}diff --git a/backend/internxt/internxt_internal_test.go b/backend/internxt/internxt_internal_test.goindex fe795cf59..a0384f244 100644--- a/backend/internxt/internxt_internal_test.go+++ b/backend/internxt/internxt_internal_test.go@@ -165,3 +165,27 @@ func TestFindFile(t *testing.T) {})}}++func TestFindFileIgnoresStale(t *testing.T) {+ ctx := context.Background()+ server := existenceServer(t, []storedFile{{"uuid-moved", "foo", "txt"}, {"uuid-deleted", "bar", "txt"}, {"uuid-renamed", "baz", "txt"}})+ defer server.Close()+ f := testFs(t, server)++ recent.moved("uuid-moved", "other-dir-uuid", "foo.txt.bak")+ recent.deleted("uuid-deleted")+ recent.moved("uuid-renamed", "dir-uuid", "baz.txt")++ file, err := f.findFile(ctx, "foo.txt", "dir-uuid")+ require.NoError(t, err)+ assert.Nil(t, file, "moved away")++ file, err = f.findFile(ctx, "bar.txt", "dir-uuid")+ require.NoError(t, err)+ assert.Nil(t, file, "deleted")++ file, err = f.findFile(ctx, "baz.txt", "dir-uuid")+ require.NoError(t, err)+ require.NotNil(t, file, "moved here")+ assert.Equal(t, "uuid-renamed", file.UUID)+}