Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

internxt: fix sync with --backup-dir or --suffix deleting the backed up file

Nick Craig-Woodf068abd607f8
diff --git a/backend/internxt/internxt.go b/backend/internxt/internxt.go
index 89f144c64..598384439 100644
--- a/backend/internxt/internxt.go
+++ b/backend/internxt/internxt.go
@@ -592,6 +592,73 @@ func existenceCheck(name, ext string) files.FileExistenceCheck {
}
}
+// staleWindow is how long the API may carry on returning an item from
+// where it was before this process moved or deleted it.
+const staleWindow = time.Minute
+
+// location is where this process last put an item. An empty parentUUID
+// means the item was deleted.
+type location struct {
+ parentUUID string
+ name string // encoded name, including any extension
+ expires time.Time
+}
+
+// recentChanges holds the files and folders this process has moved or
+// deleted recently, keyed by UUID.
+//
+// The API serves lookups and listings from read replicas which lag behind
+// writes, so an item can be returned from its old location for a while after
+// it was moved or deleted. Acting on such a stale entry can be destructive,
+// eg overwriting it would delete the file from where it was moved to.
+type recentChanges struct {
+ mu sync.Mutex
+ items map[string]location
+ lastPrune time.Time
+}
+
+// recent is shared by all the Fs in the process as UUIDs are global.
+var recent = recentChanges{items: map[string]location{}}
+
+// set records the new location of uuid, pruning expired entries.
+func (r *recentChanges) set(uuid string, loc location) {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ now := time.Now()
+ if now.Sub(r.lastPrune) > staleWindow {
+ for k, v := range r.items {
+ if now.After(v.expires) {
+ delete(r.items, k)
+ }
+ }
+ r.lastPrune = now
+ }
+ loc.expires = now.Add(staleWindow)
+ r.items[uuid] = loc
+}
+
+// moved records that uuid was moved to name in parentUUID.
+func (r *recentChanges) moved(uuid, parentUUID, name string) {
+ r.set(uuid, location{parentUUID: parentUUID, name: name})
+}
+
+// deleted records that uuid was deleted.
+func (r *recentChanges) deleted(uuid string) {
+ r.set(uuid, location{})
+}
+
+// stale reports whether an API result saying uuid is called name in
+// parentUUID is out of date because of a change made by this process.
+func (r *recentChanges) stale(uuid, parentUUID, name string) bool {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ loc, ok := r.items[uuid]
+ if !ok || time.Now().After(loc.expires) {
+ return false
+ }
+ return loc.parentUUID != parentUUID || loc.name != name
+}
+
// findFile looks up a single file by name within directoryID. Returns
// (nil, nil) when the file does not exist; surfaces transport/API errors.
func (f *Fs) findFile(ctx context.Context, leaf, directoryID string) (*folders.File, error) {
@@ -623,6 +690,10 @@ func (f *Fs) findFile(ctx context.Context, leaf, directoryID string) (*folders.F
if !result.FileExists() || result.UUID == "" {
continue
}
+ if recent.stale(result.UUID, directoryID, joinNameExt(result.PlainName, result.Type)) {
+ fs.Debugf(f, "Ignoring stale lookup of %q (UUID: %s) which was moved or deleted", leaf, result.UUID)
+ continue
+ }
if joinNameExt(result.PlainName, result.Type) == encodedName {
uuid = result.UUID
break
@@ -699,6 +770,9 @@ func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
if len(e.Type) > 0 {
remote += "." + e.Type
}
+ if recent.stale(e.UUID, dirID, remote) {
+ continue
+ }
remote = path.Join(dir, f.opt.Encoding.ToStandardName(remote))
out = append(out, newObjectWithFile(f, remote, &e))
}
@@ -905,6 +979,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
if err != nil {
return nil, err
}
+ recent.moved(srcObj.uuid, directoryID, joinNameExt(newName, newType))
dstObj := &Object{
f: f,
@@ -1124,6 +1199,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
backupName, backupType, backupUUID, err)
// Don't fail the upload just because backup deletion failed
} else {
+ recent.deleted(backupUUID)
fs.Debugf(o.f, "Successfully deleted backup file")
}
}
@@ -1229,8 +1305,13 @@ func (o *Object) restoreBackupFile(ctx context.Context, backupUUID, origName, or
// Remove deletes a file
func (o *Object) Remove(ctx context.Context) error {
- return o.f.pacer.Call(func() (bool, error) {
+ err := o.f.pacer.Call(func() (bool, error) {
err := files.DeleteFile(ctx, o.f.cfg, o.uuid)
return o.f.shouldRetry(ctx, err)
})
+ if err != nil {
+ return err
+ }
+ recent.deleted(o.uuid)
+ return nil
}
diff --git a/backend/internxt/internxt_internal_test.go b/backend/internxt/internxt_internal_test.go
index fe795cf59..a0384f244 100644
--- a/backend/internxt/internxt_internal_test.go
+++ b/backend/internxt/internxt_internal_test.go
@@ -165,3 +165,27 @@ func TestFindFile(t *testing.T) {
})
}
}
+
+func TestFindFileIgnoresStale(t *testing.T) {
+ ctx := context.Background()
+ server := existenceServer(t, []storedFile{{"uuid-moved", "foo", "txt"}, {"uuid-deleted", "bar", "txt"}, {"uuid-renamed", "baz", "txt"}})
+ defer server.Close()
+ f := testFs(t, server)
+
+ recent.moved("uuid-moved", "other-dir-uuid", "foo.txt.bak")
+ recent.deleted("uuid-deleted")
+ recent.moved("uuid-renamed", "dir-uuid", "baz.txt")
+
+ file, err := f.findFile(ctx, "foo.txt", "dir-uuid")
+ require.NoError(t, err)
+ assert.Nil(t, file, "moved away")
+
+ file, err = f.findFile(ctx, "bar.txt", "dir-uuid")
+ require.NoError(t, err)
+ assert.Nil(t, file, "deleted")
+
+ file, err = f.findFile(ctx, "baz.txt", "dir-uuid")
+ require.NoError(t, err)
+ require.NotNil(t, file, "moved here")
+ assert.Equal(t, "uuid-renamed", file.UUID)
+}