"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
onedrive: keep the delta token when a ChangeNotify poll fails - fixes #10009
xiaozou-winefa2b56200e20
diff --git a/backend/onedrive/onedrive.go b/backend/onedrive/onedrive.goindex 0df28a265..e634a495d 100644--- a/backend/onedrive/onedrive.go+++ b/backend/onedrive/onedrive.go@@ -3103,9 +3103,15 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT}case <-tickerC:fs.Debugf(f, "Checking for changes on remote")- nextDeltaToken, err = f.changeNotifyRunner(ctx, notifyFunc, nextDeltaToken)+ var newDeltaToken string+ newDeltaToken, err = f.changeNotifyRunner(ctx, notifyFunc, nextDeltaToken)if err != nil {+ // Keep the token we already have: replacing it with the+ // empty token from a failed poll would reset the delta+ // listing to the start and lose every change in between.fs.Infof(f, "Change notify listener failure: %s", err)+ } else {+ nextDeltaToken = newDeltaToken}}}@@ -3113,7 +3119,7 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT}func (f *Fs) changeNotifyStartPageToken(ctx context.Context) (nextDeltaToken string, err error) {- delta, err := f.changeNotifyNextChange(ctx, "latest")+ delta, err := f.changeNotifyNextChange(ctx, "latest", "")if err != nil {return}@@ -3125,15 +3131,34 @@ func (f *Fs) changeNotifyStartPageToken(ctx context.Context) (nextDeltaToken strreturn}-func (f *Fs) changeNotifyNextChange(ctx context.Context, token string) (delta api.DeltaResponse, err error) {- opts := f.buildDriveDeltaOpts(token)+// changeNotifyNextChange fetches one page of the drive delta listing.+//+// Pass the delta token to resume from in token, or the @odata.nextLink URL+// returned by the previous page in nextLink. Exactly one of the two should+// be set.+func (f *Fs) changeNotifyNextChange(ctx context.Context, token, nextLink string) (delta api.DeltaResponse, err error) {+ opts := f.buildDriveDeltaOpts(token, nextLink)- _, err = f.srv.CallJSON(ctx, &opts, nil, &delta)+ err = f.pacer.Call(func() (bool, error) {+ var resp *http.Response+ resp, err = f.srv.CallJSON(ctx, &opts, nil, &delta)+ return shouldRetry(ctx, resp, err)+ })return}-func (f *Fs) buildDriveDeltaOpts(token string) rest.Opts {+// buildDriveDeltaOpts builds the request for one page of the drive delta+// listing. nextLink, when set, is the complete @odata.nextLink URL from the+// previous page and is used as-is, overriding the drive root.+func (f *Fs) buildDriveDeltaOpts(token, nextLink string) rest.Opts {+ if nextLink != "" {+ return rest.Opts{+ Method: "GET",+ RootURL: nextLink,+ }+ }+var rootURL stringif f.opt.TenantURL != "" {rootURL = tenantAPIEndpoint(f.opt.TenantURL, f.opt.TenantAPIVersion) + "/drives"@@ -3149,46 +3174,58 @@ func (f *Fs) buildDriveDeltaOpts(token string) rest.Opts {}}-func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.EntryType), deltaToken string) (nextDeltaToken string, err error) {- delta, err := f.changeNotifyNextChange(ctx, deltaToken)- if err != nil {- return- }- parsedURL, err := url.Parse(delta.DeltaLink)- if err != nil {- return- }- nextDeltaToken = parsedURL.Query().Get("token")-- for _, item := range delta.Value {- isDriveRootFolder := item.GetParentReference().ID == ""- if isDriveRootFolder {- continue- }-- fullPath, err := getItemFullPath(&item)+// changeNotifyRunner polls for changes and calls notifyFunc for each one.+//+// It walks every page of the change set and returns the token to resume from+// next time. The returned token is only valid when err is nil: the caller+// must keep the token it passed in on error, otherwise a failed poll would+// reset the delta listing to the start and lose the changes in between.+func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.EntryType), deltaToken string) (string, error) {+ token, nextLink := deltaToken, ""+ for {+ delta, err := f.changeNotifyNextChange(ctx, token, nextLink)if err != nil {- fs.Errorf(f, "Could not get item full path: %s", err)- continue+ return "", err}- if fullPath == f.root {- continue- }+ for _, item := range delta.Value {+ isDriveRootFolder := item.GetParentReference().ID == ""+ if isDriveRootFolder {+ continue+ }- relName, insideRoot := getRelativePathInsideBase(f.root, fullPath)- if !insideRoot {- continue+ fullPath, err := getItemFullPath(&item)+ if err != nil {+ fs.Errorf(f, "Could not get item full path: %s", err)+ continue+ }++ if fullPath == f.root {+ continue+ }++ relName, insideRoot := getRelativePathInsideBase(f.root, fullPath)+ if !insideRoot {+ continue+ }++ if item.GetFile() != nil {+ notifyFunc(relName, fs.EntryObject)+ } else if item.GetFolder() != nil {+ notifyFunc(relName, fs.EntryDirectory)+ }}- if item.GetFile() != nil {- notifyFunc(relName, fs.EntryObject)- } else if item.GetFolder() != nil {- notifyFunc(relName, fs.EntryDirectory)+ if delta.NextLink == "" {+ // end of the change set - the deltaLink carries the resume token+ parsedURL, err := url.Parse(delta.DeltaLink)+ if err != nil {+ return "", err+ }+ return parsedURL.Query().Get("token"), nil}+ nextLink = delta.NextLink}-- return}func getItemFullPath(item *api.Item) (fullPath string, err error) {diff --git a/backend/onedrive/onedrive_internal_test.go b/backend/onedrive/onedrive_internal_test.goindex 566e6157d..88d8b34f7 100644--- a/backend/onedrive/onedrive_internal_test.go+++ b/backend/onedrive/onedrive_internal_test.go@@ -4,8 +4,11 @@ import ("context""encoding/json""fmt"+ "net/http"+ "net/http/httptest""slices""strings"+ "sync/atomic""testing""time"@@ -15,7 +18,9 @@ import ("github.com/rclone/rclone/fs/operations""github.com/rclone/rclone/fstest""github.com/rclone/rclone/fstest/fstests"+ "github.com/rclone/rclone/lib/pacer""github.com/rclone/rclone/lib/random"+ "github.com/rclone/rclone/lib/rest""github.com/stretchr/testify/assert""github.com/stretchr/testify/require")@@ -608,3 +613,131 @@ func (f *Fs) InternalTest(t *testing.T) {}var _ fstests.InternalTester = (*Fs)(nil)++// newChangeNotifyTestFs returns an Fs whose delta listing is served by+// handler on a local test server, plus the server itself. The tenant URL+// is pointed at the server so no request leaves the test.+func newChangeNotifyTestFs(t *testing.T, handler http.Handler) (*Fs, *httptest.Server) {+ t.Helper()+ server := httptest.NewServer(handler)+ t.Cleanup(server.Close)++ ctx := context.Background()+ f := &Fs{+ ci: fs.GetConfig(ctx),+ root: "",+ driveID: "testdrive",+ srv: rest.NewClient(server.Client()).SetRoot(server.URL),+ pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(10*time.Millisecond))),+ }+ f.opt.TenantURL = server.URL+ f.opt.TenantAPIVersion = defaultTenantAPIVersion+ return f, server+}++// deltaItem builds a delta entry for a file named name inside a folder.+func deltaItem(name, folder string) api.Item {+ return api.Item{+ ID: name,+ Name: name,+ File: &api.FileFacet{},+ ParentReference: &api.ItemReference{+ ID: "parentid",+ Path: "driveid:/" + folder,+ },+ }+}++// TestChangeNotifyRetriesAndKeepsToken covers a transient failure of the+// delta poll: the request must go through the pacer so it is retried, and+// a poll which fails outright must not hand back an empty token, since the+// caller stores whatever it returns. An empty token would reset the delta+// listing to the start and silently lose every change in between.+func TestChangeNotifyRetriesAndKeepsToken(t *testing.T) {+ ctx := context.Background()+ var attempts atomic.Int32+ var tokens []string+ fail := true+ f, _ := newChangeNotifyTestFs(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ attempts.Add(1)+ tokens = append(tokens, r.URL.Query().Get("token"))+ if fail {+ w.Header().Set("Content-Type", "application/json")+ w.WriteHeader(http.StatusServiceUnavailable)+ _ = json.NewEncoder(w).Encode(map[string]any{"error": map[string]any{+ "code": "serviceNotAvailable",+ "message": "Service unavailable",+ }})+ return+ }+ w.Header().Set("Content-Type", "application/json")+ _ = json.NewEncoder(w).Encode(api.DeltaResponse{+ Value: []api.Item{deltaItem("file1.txt", "folder")},+ DeltaLink: "https://example.com/delta?token=newtoken",+ })+ }))++ var notified []string+ notifyFunc := func(path string, entryType fs.EntryType) {+ notified = append(notified, path)+ }++ // The poll fails, but is retried by the pacer first+ token, err := f.changeNotifyRunner(ctx, notifyFunc, "tok1")+ require.Error(t, err)+ assert.Empty(t, token, "a failed poll must not return a token")+ assert.Equal(t, fs.GetConfig(ctx).LowLevelRetries, int(attempts.Load()), "the failed poll should have been retried by the pacer")+ assert.Equal(t, slices.Repeat([]string{"tok1"}, int(attempts.Load())), tokens, "every retry should resume from the same token")+ assert.Empty(t, notified)++ // Keeping the previous token (as ChangeNotify does) resumes correctly+ fail = false+ attempts.Store(0)+ tokens = nil+ token, err = f.changeNotifyRunner(ctx, notifyFunc, "tok1")+ require.NoError(t, err)+ assert.Equal(t, "newtoken", token)+ assert.Equal(t, []string{"tok1"}, tokens, "the second poll should resume from the retained token")+ assert.Equal(t, []string{"folder/file1.txt"}, notified)+}++// TestChangeNotifyFollowsNextLink covers a change set which spans more than+// one page. Graph returns @odata.nextLink and no @odata.deltaLink until the+// last page, so a runner which stops after the first page both loses the+// remaining changes and ends up with an empty resume token.+func TestChangeNotifyFollowsNextLink(t *testing.T) {+ ctx := context.Background()+ var urls []string+ var server *httptest.Server+ f, server := newChangeNotifyTestFs(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ urls = append(urls, r.URL.RequestURI())+ w.Header().Set("Content-Type", "application/json")+ switch len(urls) {+ case 1:+ _ = json.NewEncoder(w).Encode(api.DeltaResponse{+ Value: []api.Item{deltaItem("page1.txt", "folder")},+ NextLink: server.URL + "/nextpage?$skiptoken=abc",+ })+ case 2:+ _ = json.NewEncoder(w).Encode(api.DeltaResponse{+ Value: []api.Item{deltaItem("page2.txt", "folder")},+ DeltaLink: "https://example.com/delta?token=lasttoken",+ })+ default:+ t.Errorf("unexpected extra request to %s", r.URL)+ }+ }))++ var notified []string+ token, err := f.changeNotifyRunner(ctx, func(path string, entryType fs.EntryType) {+ notified = append(notified, path)+ }, "tok1")+ require.NoError(t, err)++ assert.Equal(t, "lasttoken", token, "the resume token should come from the last page")+ assert.Equal(t, []string{"folder/page1.txt", "folder/page2.txt"}, notified, "changes from every page should be notified")+ require.Len(t, urls, 2, "the nextLink page should have been fetched")+ assert.Contains(t, urls[0], "/testdrive/root/delta")+ assert.Contains(t, urls[0], "token=tok1")+ assert.Equal(t, "/nextpage?$skiptoken=abc", urls[1], "the nextLink should be used as-is, without the delta token")+}