🎧 Your Personal Streaming Service
Go airsonicmadsonicmedia-servermusicmusic-servernavidromeopensubsonicraspberry-piself-hostedstreamerstreaming-apistreaming-audiosubsonicsubsonic-server
perf(artwork): cap the stale-absent recheck at 100 items per kind per hour (#6007)
Deluan Quintão07b6411c0bc3
diff --git a/cmd/artwork.go b/cmd/artwork.goindex b193abeaf..01de071bb 100644--- a/cmd/artwork.go+++ b/cmd/artwork.go@@ -238,7 +238,8 @@ func formatStatus(rep statusReport) string {for _, a := range rep.absent {fmt.Fprintf(w, " %s\t%d\t%d\n", a.kind, a.Total, a.Stale)}- fmt.Fprintf(w, " (rechecked once the last attempt is older than %gh)\n", artwork.StaleAbsentAge.Hours())+ fmt.Fprintf(w, " (eligible once the last attempt is older than %gh; re-queued %d per kind per hour, oldest first)\n",+ artwork.StaleAbsentAge.Hours(), artwork.StaleAbsentRecheckBatch)fmt.Fprintln(w, "\nBackfill")fmt.Fprintf(w, " State:\t%s\n", backfillState(rep))diff --git a/cmd/artwork_test.go b/cmd/artwork_test.goindex 1d2bdae73..cc63ed86f 100644--- a/cmd/artwork_test.go+++ b/cmd/artwork_test.go@@ -818,7 +818,7 @@ var _ = Describe("collectStatus", func() {ImageType: model.ImageTypePrimary, Source: source, Hash: hash, AttemptedAt: attempted})).To(Succeed())}put(model.KindArtistArtwork, "ar-1", "external:deezer", "h1", time.Now())- put(model.KindArtistArtwork, "ar-2", "", "", time.Now().Add(-48*time.Hour))+ put(model.KindArtistArtwork, "ar-2", "", "", time.Now().Add(-artwork.StaleAbsentAge-time.Hour))put(model.KindArtistArtwork, "ar-3", "", "", time.Now())put(model.KindAlbumArtwork, "al-1", "folder", "h2", time.Now())Expect(queue.Enqueue(model.ArtworkQueueItem{ItemKind: "ar", ItemID: "ar-9",@@ -909,8 +909,9 @@ var _ = Describe("formatStatus", func() {Expect(absent).To(MatchRegexp(`artist\s+2\s+1`))})- It("states the recheck window the absent counts are bucketed against", func() {- Expect(formatStatus(rep)).To(ContainSubstring("24h"))+ It("states the recheck window and the drip rate the absent counts are bucketed against", func() {+ Expect(formatStatus(rep)).To(ContainSubstring("168h"))+ Expect(formatStatus(rep)).To(ContainSubstring("100 per kind per hour"))})It("leads with the queued backlog, which is the finding, not with the fingerprint verdict", func() {diff --git a/core/artwork/housekeeping.go b/core/artwork/housekeeping.goindex f2996d044..fed5757c8 100644--- a/core/artwork/housekeeping.go+++ b/core/artwork/housekeeping.go@@ -18,7 +18,11 @@ import ()// StaleAbsentAge is how long an absent state is trusted before a recheck retries it.-const StaleAbsentAge = 24 * time.Hour+const StaleAbsentAge = 7 * 24 * time.Hour++// StaleAbsentRecheckBatch caps how many absent states each hourly tick re-queues per kind,+// oldest first, so external agents see a flat drip instead of a daily burst.+const StaleAbsentRecheckBatch = 100// RecheckKinds omits media files: they resolve embedded-only, at scan or on view.var RecheckKinds = []model.Kind{@@ -125,7 +129,7 @@ func enqueueStaleAbsentAll(ctx context.Context, ds model.DataStore) error {cutoff := time.Now().Add(-StaleAbsentAge)queue := ds.ArtworkQueue(ctx)for _, kind := range RecheckKinds {- if _, err := queue.EnqueueStaleAbsent(kind, cutoff); err != nil {+ if _, err := queue.EnqueueStaleAbsent(kind, cutoff, StaleAbsentRecheckBatch); err != nil {return err}}diff --git a/core/artwork/housekeeping_test.go b/core/artwork/housekeeping_test.goindex 4ea15ab04..4f229bff8 100644--- a/core/artwork/housekeeping_test.go+++ b/core/artwork/housekeeping_test.go@@ -2,6 +2,7 @@ package artworkimport ("context"+ "fmt""slices""time"@@ -235,8 +236,8 @@ var _ = Describe("Housekeeping", func() {})It("enqueues only absent entries older than the recheck window, across all kinds", func() {- old := time.Now().Add(-48 * time.Hour)- recent := time.Now().Add(-time.Hour)+ old := time.Now().Add(-StaleAbsentAge - time.Hour)+ recent := time.Now().Add(-StaleAbsentAge + time.Hour)artRepo.ItemData["ar-stale"] = model.ItemArtwork{ItemKind: "ar", ItemID: "ar1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old}artRepo.ItemData["al-stale"] = model.ItemArtwork{ItemKind: "al", ItemID: "al1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old}@@ -259,6 +260,20 @@ var _ = Describe("Housekeeping", func() {Expect(findQueued(queueRepo.MockArtworkQueueRepo, "ar", "ar2")).To(BeNil())Expect(findQueued(queueRepo.MockArtworkQueueRepo, "al", "al2")).To(BeNil())})++ It("caps each tick at the recheck batch, oldest attempts first", func() {+ for i := range StaleAbsentRecheckBatch + 1 {+ id := fmt.Sprintf("ar%d", i)+ artRepo.ItemData[id] = model.ItemArtwork{ItemKind: "ar", ItemID: id, ImageType: model.ImageTypePrimary,+ Hash: "", AttemptedAt: time.Now().Add(-StaleAbsentAge - time.Duration(i+1)*time.Minute)}+ }++ Expect(enqueueStaleAbsentAll(ctx, ds)).To(Succeed())++ Expect(queueRepo.Data).To(HaveLen(StaleAbsentRecheckBatch))+ // ar0 has the newest attempted_at of the cohort, so it is the one left out.+ Expect(findQueued(queueRepo.MockArtworkQueueRepo, "ar", "ar0")).To(BeNil())+ })})Describe("EnqueueMissingAll", func() {diff --git a/core/artwork/worker.go b/core/artwork/worker.goindex 6271e54bf..0f947f6d7 100644--- a/core/artwork/worker.go+++ b/core/artwork/worker.go@@ -137,7 +137,8 @@ func (w *Worker) Backfill(ctx context.Context) (bool, error) {return backfill(ctx, w.proc.ds)}-// EnqueueStaleAbsentAll requeues known-absent entries older than StaleAbsentAge.+// EnqueueStaleAbsentAll requeues known-absent entries older than StaleAbsentAge, at most+// StaleAbsentRecheckBatch per kind, oldest first.func (w *Worker) EnqueueStaleAbsentAll(ctx context.Context) error {return enqueueStaleAbsentAll(ctx, w.proc.ds)}diff --git a/model/artwork.go b/model/artwork.goindex b484b0500..3c0df209b 100644--- a/model/artwork.go+++ b/model/artwork.go@@ -134,8 +134,9 @@ type ArtworkQueueRepository interface {// EnqueuePreservingBackoff upserts like Enqueue but preserves an existing row's retry_at, so a// request-triggered read-through never resets a failed resolution's backoff.EnqueuePreservingBackoff(items ...ArtworkQueueItem) error- // EnqueueStaleAbsent inserts queue rows (priority Recheck) for absent states older than cutoff.- EnqueueStaleAbsent(kind Kind, attemptedBefore time.Time) (int64, error)+ // EnqueueStaleAbsent inserts queue rows (priority Recheck) for absent states older than cutoff, oldest+ // first; limit caps the selection, so already-queued rows use up budget (backpressure when the drain stalls).+ EnqueueStaleAbsent(kind Kind, attemptedBefore time.Time, limit int) (int64, error)// EnqueueAllMissing inserts queue rows for all entities with no item_artwork row, at the given priority.EnqueueAllMissing(kind Kind, priority int) (int64, error)// EnqueueIfMissing inserts only for items with no item_artwork row yet.@@ -160,8 +161,8 @@ type ArtworkQueueRepository interface {// CountQueued reports the pending rows matching the kinds and priorities, grouped by both;// an empty filter means every one.CountQueued(kinds []Kind, priorities []int) ([]ArtworkQueueStat, error)- // CountAbsent reports the absent states of a kind, and how many of those EnqueueStaleAbsent- // would pick up at the given cutoff.+ // CountAbsent reports the absent states of a kind, and how many are past the given cutoff,+ // eligible for EnqueueStaleAbsent (which drains them limit rows per call).CountAbsent(kind Kind, attemptedBefore time.Time) (ArtworkAbsentStat, error)// PurgeDangling removes queue rows whose entity no longer exists.PurgeDangling() (int64, error)diff --git a/persistence/artwork_queue_repository.go b/persistence/artwork_queue_repository.goindex 5db122a8b..1c0077fc7 100644--- a/persistence/artwork_queue_repository.go+++ b/persistence/artwork_queue_repository.go@@ -56,11 +56,12 @@ func (r *artworkQueueRepository) EnqueuePreservingBackoff(items ...model.Artworkpriority = MAX(priority, excluded.priority)`, items)}-func (r *artworkQueueRepository) EnqueueStaleAbsent(kind model.Kind, attemptedBefore time.Time) (int64, error) {+func (r *artworkQueueRepository) EnqueueStaleAbsent(kind model.Kind, attemptedBefore time.Time, limit int) (int64, error) {now := time.Now()return r.insertIfNotQueued("", `SELECT item_kind, item_id, image_type, ?, 0, ?, ?- FROM `+itemArtworkTable+` WHERE item_kind = ? AND hash = '' AND attempted_at < ?`,- model.ArtworkPriorityRecheck, now, now, kind.Prefix(), attemptedBefore)+ FROM `+itemArtworkTable+` WHERE item_kind = ? AND hash = '' AND attempted_at < ?+ ORDER BY attempted_at LIMIT ?`,+ model.ArtworkPriorityRecheck, now, now, kind.Prefix(), attemptedBefore, limit)}func (r *artworkQueueRepository) EnqueueAllMissing(kind model.Kind, priority int) (int64, error) {@@ -230,7 +231,7 @@ func (r *artworkQueueRepository) Count() (int64, error) {return res.Count, err}-// CountAbsent matches EnqueueStaleAbsent on hash, so the stale count is what a recheck would queue.+// CountAbsent matches EnqueueStaleAbsent on hash, so the stale count is the pool a recheck drains from.func (r *artworkQueueRepository) CountAbsent(kind model.Kind, attemptedBefore time.Time) (model.ArtworkAbsentStat, error) {var res model.ArtworkAbsentStaterr := r.queryOne(Select("count(*) as total").diff --git a/persistence/artwork_queue_repository_test.go b/persistence/artwork_queue_repository_test.goindex 35eebdd6d..1673a5b0e 100644--- a/persistence/artwork_queue_repository_test.go+++ b/persistence/artwork_queue_repository_test.go@@ -244,7 +244,7 @@ var _ = Describe("ArtworkQueueRepository", func() {Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "fresh1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: time.Now()})).To(Succeed())Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "found1", ImageType: model.ImageTypePrimary, Hash: "hX", AttemptedAt: old})).To(Succeed())- n, err := repo.EnqueueStaleAbsent(model.KindArtistArtwork, time.Now().Add(-24*time.Hour))+ n, err := repo.EnqueueStaleAbsent(model.KindArtistArtwork, time.Now().Add(-24*time.Hour), 100)Expect(err).ToNot(HaveOccurred())Expect(n).To(Equal(int64(1)))@@ -255,6 +255,23 @@ var _ = Describe("ArtworkQueueRepository", func() {Expect(items[0].Priority).To(Equal(model.ArtworkPriorityRecheck))})+ It("enqueues only the oldest stale absent states up to the limit", func() {+ awRepo := NewArtworkRepository(context.Background(), GetDBXBuilder())+ now := time.Now()+ Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "oldest", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: now.Add(-72 * time.Hour)})).To(Succeed())+ Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "older", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: now.Add(-60 * time.Hour)})).To(Succeed())+ Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "old", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: now.Add(-48 * time.Hour)})).To(Succeed())++ n, err := repo.EnqueueStaleAbsent(model.KindArtistArtwork, now.Add(-24*time.Hour), 2)+ Expect(err).ToNot(HaveOccurred())+ Expect(n).To(Equal(int64(2)))++ items, err := repo.DequeueBatch(10)+ Expect(err).ToNot(HaveOccurred())+ ids := slice.Map(items, func(it model.ArtworkQueueItem) string { return it.ItemID })+ Expect(ids).To(ConsistOf("oldest", "older"))+ })+It("enqueues entities that have no item_artwork row at all", func() {awRepo := NewArtworkRepository(context.Background(), GetDBXBuilder())Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: albumSgtPeppers.ID, ImageType: model.ImageTypePrimary, Hash: "hX", AttemptedAt: time.Now()})).To(Succeed())diff --git a/tests/mock_artwork_queue_repo.go b/tests/mock_artwork_queue_repo.goindex 70f7084ff..51ddf4b61 100644--- a/tests/mock_artwork_queue_repo.go+++ b/tests/mock_artwork_queue_repo.go@@ -272,18 +272,24 @@ func (m *MockArtworkQueueRepo) EnqueuePreservingBackoff(items ...model.ArtworkQureturn nil}-func (m *MockArtworkQueueRepo) EnqueueStaleAbsent(kind model.Kind, attemptedBefore time.Time) (int64, error) {+func (m *MockArtworkQueueRepo) EnqueueStaleAbsent(kind model.Kind, attemptedBefore time.Time, limit int) (int64, error) {m.mu.Lock()defer m.mu.Unlock()if m.Err != nil || m.ItemArtworkSource == nil {return 0, m.Err}- now := time.Now()- var inserted int64+ var stale []model.ItemArtworkfor _, ia := range m.ItemArtworkSource.ItemData {- if ia.ItemKind != kind.Prefix() || ia.Hash != "" || !ia.AttemptedAt.Before(attemptedBefore) {- continue+ if ia.ItemKind == kind.Prefix() && ia.Hash == "" && ia.AttemptedAt.Before(attemptedBefore) {+ stale = append(stale, ia)}+ }+ slices.SortFunc(stale, func(a, b model.ItemArtwork) int { return a.AttemptedAt.Compare(b.AttemptedAt) })+ // The limit caps the selection, like the SQL's LIMIT before ON CONFLICT: queued rows use up budget.+ stale = stale[:min(limit, len(stale))]+ now := time.Now()+ var inserted int64+ for _, ia := range stale {k := iaKey(ia.ItemKind, ia.ItemID, ia.ImageType)if _, ok := m.Data[k]; ok { // DO NOTHING: never touch existing queue rowscontinue