"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
zoho: log throttling once per episode at NOTICE
Erol Ozcan1daa03f1085b
diff --git a/backend/zoho/throttle_test.go b/backend/zoho/throttle_test.goindex 17e95eafb..b73ab4187 100644--- a/backend/zoho/throttle_test.go+++ b/backend/zoho/throttle_test.go@@ -3,6 +3,7 @@ package zohoimport ("context""net/http"+ "sync""testing""time"@@ -11,10 +12,13 @@ import ("github.com/stretchr/testify/require")-// newTestFs returns a bare *Fs so the 429/retry logic in shouldRetry can be-// exercised in isolation. No network or pacer is involved.+// newTestFs returns a bare *Fs carrying just the throttle state shouldRetry+// touches, armed exactly as NewFs arms it. No network or pacer is involved, so+// the 429/retry logic can be exercised in isolation.func newTestFs() *Fs {- return &Fs{}+ f := &Fs{throttle: &throttleState{}}+ f.throttle.progress.Store(true)+ return f}func TestShouldRetry(t *testing.T) {@@ -78,3 +82,52 @@ func TestShouldRetry(t *testing.T) {assert.False(t, retry)})}++// TestThrottleEpisode covers the once-per-episode logging state machine that+// logThrottle/shouldRetry drive through throttleState, without sleeping: the+// penalty window is moved by hand instead of waited out.+func TestThrottleEpisode(t *testing.T) {+ ctx := context.Background()+ f := newTestFs() // progress armed: the first 429 would log at NOTICE+ resp429 := &http.Response{StatusCode: 429, Header: http.Header{"Retry-After": {"1"}}}+ respOK := &http.Response{StatusCode: 200}++ // The first 429 consumes the armed flag (logs once) and opens a penalty window.+ _, _ = f.shouldRetry(ctx, resp429, assert.AnError)+ assert.False(t, f.throttle.progress.Load(), "first 429 disarms progress")++ // A success still inside the penalty window must not re-arm (would let a+ // burst of in-flight successes start a fresh episode too early).+ _, _ = f.shouldRetry(ctx, respOK, nil)+ assert.False(t, f.throttle.progress.Load(), "success during penalty window does not re-arm")++ // Once the penalty window has elapsed, a success re-arms for the next episode.+ f.throttle.penaltyUntilNano.Store(time.Now().Add(-time.Second).UnixNano())+ _, _ = f.shouldRetry(ctx, respOK, nil)+ assert.True(t, f.throttle.progress.Load(), "success after penalty window re-arms")+}++// TestThrottleStateConcurrent drives the lock-free throttleState (shared across+// the shallow Fs copy) from many goroutines. Like the registry test, `go test+// -race` is the real assertion: it would flag any non-atomic access. The final+// flag is interleaving-dependent, so it is deliberately not asserted.+func TestThrottleStateConcurrent(t *testing.T) {+ ctx := context.Background()+ f := newTestFs()+ resp429 := &http.Response{StatusCode: 429, Header: http.Header{"Retry-After": {"1"}}}+ respOK := &http.Response{StatusCode: 200}++ var wg sync.WaitGroup+ for i := range 64 {+ wg.Add(1)+ go func(i int) {+ defer wg.Done()+ if i%2 == 0 {+ _, _ = f.shouldRetry(ctx, resp429, assert.AnError)+ } else {+ _, _ = f.shouldRetry(ctx, respOK, nil)+ }+ }(i)+ }+ wg.Wait()+}diff --git a/backend/zoho/zoho.go b/backend/zoho/zoho.goindex 89ab21326..d35be9226 100644--- a/backend/zoho/zoho.go+++ b/backend/zoho/zoho.go@@ -12,6 +12,7 @@ import ("path""strconv""strings"+ "sync/atomic""time""github.com/google/uuid"@@ -291,6 +292,17 @@ type Fs struct {uploadsrv *rest.Client // the connection to the upload serverdirCache *dircache.DirCache // Map of directory path to directory idpacer *fs.Pacer // pacer for API calls+ throttle *throttleState // once-per-episode 429 logging state (shared across Fs copies)+}++// throttleState tracks 429 throttling for once-per-episode logging (see+// logThrottle). It sits behind a pointer so the "assume it is a file" shallow+// copy of Fs shares one state; both fields are atomic, so it is lock-free. An+// "episode" is a run of 429s with no recovery (a success after the penalty+// window clears) between them.+type throttleState struct {+ penaltyUntilNano atomic.Int64 // unix-nanos the current 429 penalty should clear+ progress atomic.Bool // a call succeeded after the last penalty window}// Object describes a Zoho WorkDrive object@@ -421,17 +433,41 @@ var retryErrorCodes = []int{509, // Bandwidth Limit Exceeded}+// logThrottle logs the first 429 in a throttling episode at NOTICE level.+//+// The err value contains the server response body. Further 429s are not logged+// until shouldRetry observes a real recovery, which prevents sustained+// throttling from flooding the log. The pacer still logs every retry at DEBUG.+// Using recovery instead of a fixed time window works with both Zoho penalty+// regimes.+func (f *Fs) logThrottle(wait time.Duration, err error) {+ newBurst := f.throttle.progress.Swap(false)+ f.throttle.penaltyUntilNano.Store(time.Now().Add(wait).UnixNano())+ secs := int(wait / time.Second)+ if newBurst {+ fs.Logf(f, "Too many requests: Trying again in %d seconds. %v", secs, err)+ }+}+// shouldRetry reports whether the given resp and err deserve to be retried.//// A 429 is honoured via the Retry-After header (falling back to 60s plus a-// margin); expired OAuth tokens are retried, missing OAuth scopes abort, and-// standard HTTP retry conditions are also handled.+// margin) and starts or continues a throttling episode; expired OAuth tokens+// are retried, missing OAuth scopes abort, and standard HTTP retry conditions+// are also handled.//// Returns whether to retry, and the err as a convenience.func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {if fserrors.ContextError(ctx, &err) {return false, err}+ if err == nil && resp != nil && resp.StatusCode < 400 {+ // Treat as recovered only after the latest 429 wait ends, so in-flight+ // successes don't start a new throttling episode too early.+ if time.Now().UnixNano() > f.throttle.penaltyUntilNano.Load() {+ f.throttle.progress.Store(true)+ }+ }authRetry := false// Bail out early if we are missing OAuth Scopes.@@ -454,10 +490,12 @@ func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bfs.Logf(f, "Failed to parse Retry-After: %q: %v", values[0], parseErr)} else {wait := time.Duration(retryAfter)*time.Second + retryAfterMargin+ f.logThrottle(wait, err)return true, pacer.RetryAfterError(err, wait)}}wait := 60*time.Second + retryAfterMargin+ f.logThrottle(wait, err)return true, pacer.RetryAfterError(err, wait)}return authRetry || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err@@ -595,7 +633,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, edownloadsrv: rest.NewClient(oAuthClient).SetRoot(downloadURL),uploadsrv: rest.NewClient(oAuthClient).SetRoot(uploadURL),pacer: fs.NewPacer(ctx, pacer.NewGoogleDrive(pacer.MinSleep(pacerMinSleep), pacer.Burst(pacerBurst))),+ throttle: &throttleState{},}+ // Arm progress so the very first 429 is logged at NOTICE.+ f.throttle.progress.Store(true)f.features = (&fs.Features{CanHaveEmptyDirectories: true,}).Fill(ctx, f)