Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

onedrive: keep the delta token when a ChangeNotify poll fails - fixes #10009

xiaozou-winefa2b56200e20
diff --git a/backend/onedrive/onedrive.go b/backend/onedrive/onedrive.go
index 0df28a265..e634a495d 100644
--- a/backend/onedrive/onedrive.go
+++ b/backend/onedrive/onedrive.go
@@ -3103,9 +3103,15 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
}
case <-tickerC:
fs.Debugf(f, "Checking for changes on remote")
- nextDeltaToken, err = f.changeNotifyRunner(ctx, notifyFunc, nextDeltaToken)
+ var newDeltaToken string
+ newDeltaToken, err = f.changeNotifyRunner(ctx, notifyFunc, nextDeltaToken)
if err != nil {
+ // Keep the token we already have: replacing it with the
+ // empty token from a failed poll would reset the delta
+ // listing to the start and lose every change in between.
fs.Infof(f, "Change notify listener failure: %s", err)
+ } else {
+ nextDeltaToken = newDeltaToken
}
}
}
@@ -3113,7 +3119,7 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
}
func (f *Fs) changeNotifyStartPageToken(ctx context.Context) (nextDeltaToken string, err error) {
- delta, err := f.changeNotifyNextChange(ctx, "latest")
+ delta, err := f.changeNotifyNextChange(ctx, "latest", "")
if err != nil {
return
}
@@ -3125,15 +3131,34 @@ func (f *Fs) changeNotifyStartPageToken(ctx context.Context) (nextDeltaToken str
return
}
-func (f *Fs) changeNotifyNextChange(ctx context.Context, token string) (delta api.DeltaResponse, err error) {
- opts := f.buildDriveDeltaOpts(token)
+// changeNotifyNextChange fetches one page of the drive delta listing.
+//
+// Pass the delta token to resume from in token, or the @odata.nextLink URL
+// returned by the previous page in nextLink. Exactly one of the two should
+// be set.
+func (f *Fs) changeNotifyNextChange(ctx context.Context, token, nextLink string) (delta api.DeltaResponse, err error) {
+ opts := f.buildDriveDeltaOpts(token, nextLink)
- _, err = f.srv.CallJSON(ctx, &opts, nil, &delta)
+ err = f.pacer.Call(func() (bool, error) {
+ var resp *http.Response
+ resp, err = f.srv.CallJSON(ctx, &opts, nil, &delta)
+ return shouldRetry(ctx, resp, err)
+ })
return
}
-func (f *Fs) buildDriveDeltaOpts(token string) rest.Opts {
+// buildDriveDeltaOpts builds the request for one page of the drive delta
+// listing. nextLink, when set, is the complete @odata.nextLink URL from the
+// previous page and is used as-is, overriding the drive root.
+func (f *Fs) buildDriveDeltaOpts(token, nextLink string) rest.Opts {
+ if nextLink != "" {
+ return rest.Opts{
+ Method: "GET",
+ RootURL: nextLink,
+ }
+ }
+
var rootURL string
if f.opt.TenantURL != "" {
rootURL = tenantAPIEndpoint(f.opt.TenantURL, f.opt.TenantAPIVersion) + "/drives"
@@ -3149,46 +3174,58 @@ func (f *Fs) buildDriveDeltaOpts(token string) rest.Opts {
}
}
-func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.EntryType), deltaToken string) (nextDeltaToken string, err error) {
- delta, err := f.changeNotifyNextChange(ctx, deltaToken)
- if err != nil {
- return
- }
- parsedURL, err := url.Parse(delta.DeltaLink)
- if err != nil {
- return
- }
- nextDeltaToken = parsedURL.Query().Get("token")
-
- for _, item := range delta.Value {
- isDriveRootFolder := item.GetParentReference().ID == ""
- if isDriveRootFolder {
- continue
- }
-
- fullPath, err := getItemFullPath(&item)
+// changeNotifyRunner polls for changes and calls notifyFunc for each one.
+//
+// It walks every page of the change set and returns the token to resume from
+// next time. The returned token is only valid when err is nil: the caller
+// must keep the token it passed in on error, otherwise a failed poll would
+// reset the delta listing to the start and lose the changes in between.
+func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.EntryType), deltaToken string) (string, error) {
+ token, nextLink := deltaToken, ""
+ for {
+ delta, err := f.changeNotifyNextChange(ctx, token, nextLink)
if err != nil {
- fs.Errorf(f, "Could not get item full path: %s", err)
- continue
+ return "", err
}
- if fullPath == f.root {
- continue
- }
+ for _, item := range delta.Value {
+ isDriveRootFolder := item.GetParentReference().ID == ""
+ if isDriveRootFolder {
+ continue
+ }
- relName, insideRoot := getRelativePathInsideBase(f.root, fullPath)
- if !insideRoot {
- continue
+ fullPath, err := getItemFullPath(&item)
+ if err != nil {
+ fs.Errorf(f, "Could not get item full path: %s", err)
+ continue
+ }
+
+ if fullPath == f.root {
+ continue
+ }
+
+ relName, insideRoot := getRelativePathInsideBase(f.root, fullPath)
+ if !insideRoot {
+ continue
+ }
+
+ if item.GetFile() != nil {
+ notifyFunc(relName, fs.EntryObject)
+ } else if item.GetFolder() != nil {
+ notifyFunc(relName, fs.EntryDirectory)
+ }
}
- if item.GetFile() != nil {
- notifyFunc(relName, fs.EntryObject)
- } else if item.GetFolder() != nil {
- notifyFunc(relName, fs.EntryDirectory)
+ if delta.NextLink == "" {
+ // end of the change set - the deltaLink carries the resume token
+ parsedURL, err := url.Parse(delta.DeltaLink)
+ if err != nil {
+ return "", err
+ }
+ return parsedURL.Query().Get("token"), nil
}
+ nextLink = delta.NextLink
}
-
- return
}
func getItemFullPath(item *api.Item) (fullPath string, err error) {
diff --git a/backend/onedrive/onedrive_internal_test.go b/backend/onedrive/onedrive_internal_test.go
index 566e6157d..88d8b34f7 100644
--- a/backend/onedrive/onedrive_internal_test.go
+++ b/backend/onedrive/onedrive_internal_test.go
@@ -4,8 +4,11 @@ import (
"context"
"encoding/json"
"fmt"
+ "net/http"
+ "net/http/httptest"
"slices"
"strings"
+ "sync/atomic"
"testing"
"time"
@@ -15,7 +18,9 @@ import (
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/fstest/fstests"
+ "github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/random"
+ "github.com/rclone/rclone/lib/rest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -608,3 +613,131 @@ func (f *Fs) InternalTest(t *testing.T) {
}
var _ fstests.InternalTester = (*Fs)(nil)
+
+// newChangeNotifyTestFs returns an Fs whose delta listing is served by
+// handler on a local test server, plus the server itself. The tenant URL
+// is pointed at the server so no request leaves the test.
+func newChangeNotifyTestFs(t *testing.T, handler http.Handler) (*Fs, *httptest.Server) {
+ t.Helper()
+ server := httptest.NewServer(handler)
+ t.Cleanup(server.Close)
+
+ ctx := context.Background()
+ f := &Fs{
+ ci: fs.GetConfig(ctx),
+ root: "",
+ driveID: "testdrive",
+ srv: rest.NewClient(server.Client()).SetRoot(server.URL),
+ pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(10*time.Millisecond))),
+ }
+ f.opt.TenantURL = server.URL
+ f.opt.TenantAPIVersion = defaultTenantAPIVersion
+ return f, server
+}
+
+// deltaItem builds a delta entry for a file named name inside a folder.
+func deltaItem(name, folder string) api.Item {
+ return api.Item{
+ ID: name,
+ Name: name,
+ File: &api.FileFacet{},
+ ParentReference: &api.ItemReference{
+ ID: "parentid",
+ Path: "driveid:/" + folder,
+ },
+ }
+}
+
+// TestChangeNotifyRetriesAndKeepsToken covers a transient failure of the
+// delta poll: the request must go through the pacer so it is retried, and
+// a poll which fails outright must not hand back an empty token, since the
+// caller stores whatever it returns. An empty token would reset the delta
+// listing to the start and silently lose every change in between.
+func TestChangeNotifyRetriesAndKeepsToken(t *testing.T) {
+ ctx := context.Background()
+ var attempts atomic.Int32
+ var tokens []string
+ fail := true
+ f, _ := newChangeNotifyTestFs(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ attempts.Add(1)
+ tokens = append(tokens, r.URL.Query().Get("token"))
+ if fail {
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(http.StatusServiceUnavailable)
+ _ = json.NewEncoder(w).Encode(map[string]any{"error": map[string]any{
+ "code": "serviceNotAvailable",
+ "message": "Service unavailable",
+ }})
+ return
+ }
+ w.Header().Set("Content-Type", "application/json")
+ _ = json.NewEncoder(w).Encode(api.DeltaResponse{
+ Value: []api.Item{deltaItem("file1.txt", "folder")},
+ DeltaLink: "https://example.com/delta?token=newtoken",
+ })
+ }))
+
+ var notified []string
+ notifyFunc := func(path string, entryType fs.EntryType) {
+ notified = append(notified, path)
+ }
+
+ // The poll fails, but is retried by the pacer first
+ token, err := f.changeNotifyRunner(ctx, notifyFunc, "tok1")
+ require.Error(t, err)
+ assert.Empty(t, token, "a failed poll must not return a token")
+ assert.Equal(t, fs.GetConfig(ctx).LowLevelRetries, int(attempts.Load()), "the failed poll should have been retried by the pacer")
+ assert.Equal(t, slices.Repeat([]string{"tok1"}, int(attempts.Load())), tokens, "every retry should resume from the same token")
+ assert.Empty(t, notified)
+
+ // Keeping the previous token (as ChangeNotify does) resumes correctly
+ fail = false
+ attempts.Store(0)
+ tokens = nil
+ token, err = f.changeNotifyRunner(ctx, notifyFunc, "tok1")
+ require.NoError(t, err)
+ assert.Equal(t, "newtoken", token)
+ assert.Equal(t, []string{"tok1"}, tokens, "the second poll should resume from the retained token")
+ assert.Equal(t, []string{"folder/file1.txt"}, notified)
+}
+
+// TestChangeNotifyFollowsNextLink covers a change set which spans more than
+// one page. Graph returns @odata.nextLink and no @odata.deltaLink until the
+// last page, so a runner which stops after the first page both loses the
+// remaining changes and ends up with an empty resume token.
+func TestChangeNotifyFollowsNextLink(t *testing.T) {
+ ctx := context.Background()
+ var urls []string
+ var server *httptest.Server
+ f, server := newChangeNotifyTestFs(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ urls = append(urls, r.URL.RequestURI())
+ w.Header().Set("Content-Type", "application/json")
+ switch len(urls) {
+ case 1:
+ _ = json.NewEncoder(w).Encode(api.DeltaResponse{
+ Value: []api.Item{deltaItem("page1.txt", "folder")},
+ NextLink: server.URL + "/nextpage?$skiptoken=abc",
+ })
+ case 2:
+ _ = json.NewEncoder(w).Encode(api.DeltaResponse{
+ Value: []api.Item{deltaItem("page2.txt", "folder")},
+ DeltaLink: "https://example.com/delta?token=lasttoken",
+ })
+ default:
+ t.Errorf("unexpected extra request to %s", r.URL)
+ }
+ }))
+
+ var notified []string
+ token, err := f.changeNotifyRunner(ctx, func(path string, entryType fs.EntryType) {
+ notified = append(notified, path)
+ }, "tok1")
+ require.NoError(t, err)
+
+ assert.Equal(t, "lasttoken", token, "the resume token should come from the last page")
+ assert.Equal(t, []string{"folder/page1.txt", "folder/page2.txt"}, notified, "changes from every page should be notified")
+ require.Len(t, urls, 2, "the nextLink page should have been fetched")
+ assert.Contains(t, urls[0], "/testdrive/root/delta")
+ assert.Contains(t, urls[0], "token=tok1")
+ assert.Equal(t, "/nextpage?$skiptoken=abc", urls[1], "the nextLink should be used as-is, without the delta token")
+}