🎧 Your Personal Streaming Service
Go airsonicmadsonicmedia-servermusicmusic-servernavidromeopensubsonicraspberry-piself-hostedstreamerstreaming-apistreaming-audiosubsonicsubsonic-server
feat(scrobbler): exponential backoff for scrobble retries during outages (#5818)
Deluan Quintãoe6597398c2f9
diff --git a/core/scrobbler/buffered_scrobbler.go b/core/scrobbler/buffered_scrobbler.goindex 408ab410d..38ea83228 100644--- a/core/scrobbler/buffered_scrobbler.go+++ b/core/scrobbler/buffered_scrobbler.go@@ -10,6 +10,30 @@ import ("github.com/navidrome/navidrome/model/request")+const (+ minRetryDelay = 5 * time.Second+ maxRetryDelay = 4 * time.Minute+ // maxRetryShift caps the exponent so the shift never overflows int64.+ // minRetryDelay<<6 = 320s already exceeds maxRetryDelay, so 6 reaches the ceiling.+ maxRetryShift = 6+)++// backoffDelay returns the delay for a zero-based retry index (0 = first retry):+// minRetryDelay doubled per prior failure, clamped to maxRetryDelay.+func backoffDelay(failures int) time.Duration {+ if failures < 0 {+ failures = 0+ }+ if failures >= maxRetryShift {+ return maxRetryDelay+ }+ d := minRetryDelay << failures+ if d > maxRetryDelay {+ return maxRetryDelay+ }+ return d+}+// Loader is a function that loads a scrobbler by name.// It returns the scrobbler and true if found, or nil and false if not available.// This allows the buffered scrobbler to always get the current plugin instance.@@ -98,15 +122,23 @@ func (b *bufferedScrobbler) sendWakeSignal() {}func (b *bufferedScrobbler) run(ctx context.Context) {+ timer := time.NewTimer(time.Hour)+ timer.Stop()+ defer timer.Stop()+ failures := 0for {- if !b.processQueue(ctx) {- time.AfterFunc(5*time.Second, func() {- b.sendWakeSignal()- })+ if b.processQueue(ctx) {+ failures = 0+ timer.Stop()+ } else {+ timer.Reset(backoffDelay(failures))+ if failures < maxRetryShift {+ failures+++ }}select {case <-b.wakeSignal:- continue+ case <-timer.C:case <-ctx.Done():return}diff --git a/core/scrobbler/buffered_scrobbler_test.go b/core/scrobbler/buffered_scrobbler_test.goindex d11f0b003..c250085ef 100644--- a/core/scrobbler/buffered_scrobbler_test.go+++ b/core/scrobbler/buffered_scrobbler_test.go@@ -2,6 +2,9 @@ package scrobblerimport ("context"+ "sync/atomic"+ "testing"+ "testing/synctest""time""github.com/navidrome/navidrome/model"@@ -100,3 +103,91 @@ var _ = Describe("BufferedScrobbler", func() {}})})++var _ = Describe("backoffDelay", func() {+ DescribeTable("computes the exponential backoff curve clamped to the ceiling",+ func(failures int, expected time.Duration) {+ Expect(backoffDelay(failures)).To(Equal(expected))+ },+ Entry("first failure", 0, 5*time.Second),+ Entry("second failure", 1, 10*time.Second),+ Entry("third failure", 2, 20*time.Second),+ Entry("fourth failure", 3, 40*time.Second),+ Entry("fifth failure", 4, 80*time.Second),+ Entry("sixth failure", 5, 160*time.Second),+ Entry("reaches the ceiling", 6, 4*time.Minute),+ Entry("stays clamped past the ceiling", 7, 4*time.Minute),+ Entry("stays clamped for large values", 1000, 4*time.Minute),+ Entry("negative is treated as zero", -1, 5*time.Second),+ )+})++// Drives the real run loop and asserts the exact retry schedule + recovery. Plain+// test: testing/synctest's fake clock needs a *testing.T, which Ginkgo doesn't give.+func TestBufferedScrobblerBackoffSchedule(t *testing.T) {+ synctest.Test(t, func(t *testing.T) {+ g := NewWithT(t)+ buffer := tests.CreateMockedScrobbleBufferRepo()+ userRepo := tests.CreateMockUserRepo()+ g.Expect(userRepo.Put(&model.User{ID: "user1", UserName: "alice"})).To(Succeed())+ ds := &tests.MockDataStore{MockedScrobbleBuffer: buffer, MockedUser: userRepo}++ flaky := &recoveringScrobbler{}+ flaky.fail(ErrRetryLater)+ bs := newBufferedScrobbler(ds, flaky, "flaky")+ defer func() { bs.Stop(); synctest.Wait() }()++ // Let the loop settle on the empty buffer, then enqueue a scrobble.+ synctest.Wait()+ track := model.MediaFile{ID: "123", Title: "Test Track", Artist: "Test Artist"}+ g.Expect(bs.Scrobble(context.Background(), "user1", Scrobble{MediaFile: track, TimeStamp: time.Now()})).To(Succeed())++ // First attempt fires immediately on the enqueue wake and is left buffered.+ synctest.Wait()+ g.Expect(flaky.count.Load()).To(Equal(int32(1)))+ g.Expect(buffer.Length()).To(Equal(int64(1)))++ // Each subsequent retry waits exactly double the previous: 5s, 10s, 20s, 40s.+ for i, gap := range []time.Duration{5 * time.Second, 10 * time.Second, 20 * time.Second, 40 * time.Second} {+ want := int32(i + 2)+ time.Sleep(gap - time.Nanosecond)+ synctest.Wait()+ g.Expect(flaky.count.Load()).To(Equal(want-1), "retry fired before the %s backoff", gap)+ time.Sleep(time.Nanosecond)+ synctest.Wait()+ g.Expect(flaky.count.Load()).To(Equal(want), "retry did not fire after the %s backoff", gap)+ }++ // Once the service recovers, waking the loop drains the buffered entry.+ flaky.succeed()+ bs.sendWakeSignal()+ synctest.Wait()+ g.Expect(buffer.Length()).To(Equal(int64(0)))+ })+}++// recoveringScrobbler is a race-safe Scrobbler whose error can be toggled while+// the buffered scrobbler's goroutine is draining, to exercise retry then recovery.+type recoveringScrobbler struct {+ err atomic.Pointer[error]+ count atomic.Int32+}++func (f *recoveringScrobbler) fail(err error) { f.err.Store(&err) }+func (f *recoveringScrobbler) succeed() { f.err.Store(nil) }++func (f *recoveringScrobbler) IsAuthorized(context.Context, string) bool { return true }++func (f *recoveringScrobbler) NowPlaying(context.Context, string, *model.MediaFile, int) error {+ return nil+}++func (f *recoveringScrobbler) Scrobble(_ context.Context, _ string, _ Scrobble) error {+ f.count.Add(1)+ if e := f.err.Load(); e != nil {+ return *e+ }+ return nil+}++func (f *recoveringScrobbler) PlaybackReport(context.Context, PlaybackSession) error { return nil }