Self-hosted fitness tracker. Log workouts, track nutrition, and monitor progress. Docker deployment, mobile-friendly.
Make the catalog sync read fresh upstream data (#206)
Cartera951439f70c3
diff --git a/backend/oedb/client.go b/backend/oedb/client.goindex 678db01..26f95b9 100644--- a/backend/oedb/client.go+++ b/backend/oedb/client.go@@ -42,8 +42,12 @@ type Client struct {userAgent stringhttp *http.Client- mu sync.Mutex+ // mu is a pointer so a Revalidating view shares the lock along with the map it guards.+ mu *sync.Mutexcache map[string]*entry++ // alwaysRevalidate: never answer from cache; every read is a conditional GET.+ alwaysRevalidate bool}// entry is one cached response body plus the metadata needed to revalidate it.@@ -77,6 +81,7 @@ func New(baseURL, version string) *Client {// see the context each caller passes in.http: &http.Client{Timeout: 90 * time.Second},cache: map[string]*entry{},+ mu: &sync.Mutex{},}}@@ -254,17 +259,21 @@ func (c *Client) getJSON(ctx context.Context, path string, dst any) error {// spends most of its life in, and serving through it is what keeps a user's// search off the critical path of an upstream round trip.// - beyond both: fetch synchronously.+//+// A Revalidating view skips the first two states and always takes the third,+// conditionally.func (c *Client) get(ctx context.Context, path string) ([]byte, error) {now := time.Now()c.mu.Lock()e, ok := c.cache[path]- if ok && now.Before(e.freshFor) {+ cached := ok && !c.alwaysRevalidate+ if cached && now.Before(e.freshFor) {body := e.bodyc.mu.Unlock()return body, nil}- if ok && now.Before(e.staleFor) && len(e.body) > 0 {+ if cached && now.Before(e.staleFor) && len(e.body) > 0 {body := e.bodyif !e.refreshing {e.refreshing = true@@ -282,6 +291,17 @@ func (c *Client) get(ctx context.Context, path string) ([]byte, error) {return c.fetch(ctx, path, etag)}+// Revalidating returns a view of c that never answers from cache: every read is a+// conditional GET with the stored ETag, and a 304 reuses the cached body. It shares+// c's cache, so what it learns also refreshes what interactive reads are served.+// The catalog sync reads through it; served from cache, each pass would apply what+// the previous pass fetched, a whole interval late.+func (c *Client) Revalidating() *Client {+ v := *c+ v.alwaysRevalidate = true+ return &v+}+// revalidate refreshes a stale entry off the request path. It deliberately uses// its own timeout-bounded context rather than the caller's: the caller has// already been served, and cancelling the refresh when that request finishes@@ -442,11 +462,17 @@ func cacheWindow(header string) (fresh, stale time.Duration) {return fresh, stale}+// maxLocalePages bounds locale discovery. oedb's locale list is a curated taxonomy+// that fits on one page; a list still going after this many is a broken or hostile+// upstream, and every listed locale costs the sync a pass of its own.+const maxLocalePages = 5+// Locales lists every locale the catalog holds translated content in; oedb always-// includes en.+// includes en. The list is bounded, and a list that exceeds it is an error rather+// than a partial answer.func (c *Client) Locales(ctx context.Context) ([]string, error) {var codes []string- for page := 1; ; page++ {+ for page := 1; page <= maxLocalePages; page++ {var out struct {Locales []struct {Code string `json:"code"`@@ -460,10 +486,14 @@ func (c *Client) Locales(ctx context.Context) ([]string, error) {for _, l := range out.Locales {codes = append(codes, l.Code)}+ if out.Total > maxLocalePages*MaxPerPage || len(codes) > maxLocalePages*MaxPerPage {+ return nil, fmt.Errorf("oedb: locale list exceeds %d entries", maxLocalePages*MaxPerPage)+ }if len(out.Locales) == 0 || len(codes) >= out.Total {return codes, nil}}+ return nil, fmt.Errorf("oedb: locale list did not end within %d pages", maxLocalePages)}// Export fetches the entire published catalog in one request.diff --git a/backend/oedb/client_test.go b/backend/oedb/client_test.goindex 8aeada0..a4ac873 100644--- a/backend/oedb/client_test.go+++ b/backend/oedb/client_test.go@@ -455,3 +455,142 @@ func TestLocales_PropagatesErrors(t *testing.T) {t.Errorf("404: err = %v, want ErrNotFound", err)}}++func TestRevalidating_SendsAConditionalGetEvenWhenFresh(t *testing.T) {+ var hits, bodies int32+ var secondINM atomic.Value+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ n := atomic.AddInt32(&hits, 1)+ w.Header().Set("Cache-Control", "public, max-age=300, stale-while-revalidate=86400")+ w.Header().Set("ETag", `"v1"`)+ if n == 2 {+ secondINM.Store(r.Header.Get("If-None-Match"))+ }+ if r.Header.Get("If-None-Match") == `"v1"` {+ w.WriteHeader(http.StatusNotModified)+ return+ }+ atomic.AddInt32(&bodies, 1)+ fmt.Fprint(w, listBody("Bench Press"))+ }))+ defer srv.Close()++ v := New(srv.URL, "test").Revalidating()+ for i := 0; i < 2; i++ {+ res, err := v.List(context.Background(), ListParams{})+ if err != nil {+ t.Fatalf("list %d: %v", i, err)+ }+ if len(res.Exercises) != 1 {+ t.Fatalf("list %d: %d exercises, want 1", i, len(res.Exercises))+ }+ }+ if got := atomic.LoadInt32(&hits); got != 2 {+ t.Errorf("hits = %d, want 2: a fresh entry must still be revalidated", got)+ }+ if got, _ := secondINM.Load().(string); got != `"v1"` {+ t.Errorf("second If-None-Match = %q, want \"v1\"", got)+ }+ if got := atomic.LoadInt32(&bodies); got != 1 {+ t.Errorf("full bodies served = %d, want 1", got)+ }+}++func TestRevalidating_SharesTheCacheButInteractiveReadsStillServeStale(t *testing.T) {+ var hits int32+ release := make(chan struct{})+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ n := atomic.AddInt32(&hits, 1)+ if n >= 3 {+ <-release+ }+ v := "v1"+ if n >= 2 {+ v = "v2"+ }+ w.Header().Set("Cache-Control", "public, max-age=0, stale-while-revalidate=86400")+ w.Header().Set("ETag", `"`+v+`"`)+ fmt.Fprint(w, listBody(v))+ }))+ defer srv.Close()+ defer close(release)++ c := New(srv.URL, "test")+ first, err := c.List(context.Background(), ListParams{})+ if err != nil || first.Exercises[0].Name() != "v1" {+ t.Fatalf("prime: %v %+v", err, first)+ }+ second, err := c.Revalidating().List(context.Background(), ListParams{})+ if err != nil || second.Exercises[0].Name() != "v2" {+ t.Fatalf("revalidating read must return the new body synchronously: %v %+v", err, second)+ }++ done := make(chan string, 1)+ go func() {+ res, err := c.List(context.Background(), ListParams{})+ if err != nil {+ done <- err.Error()+ return+ }+ done <- res.Exercises[0].Name()+ }()+ select {+ case got := <-done:+ if got != "v2" {+ t.Errorf("interactive read = %q, want the view's v2 from the shared cache", got)+ }+ case <-time.After(2 * time.Second):+ t.Fatal("interactive read waited on the network instead of serving stale")+ }+ if got := atomic.LoadInt32(&hits); got > 3 {+ t.Errorf("hits = %d, want at most 2 synchronous plus 1 background", got)+ }+}++func TestLocales_BoundedAgainstHostileLists(t *testing.T) {+ cases := []struct {+ name string+ total int+ perPage func(page int) int+ wantErr bool+ wantHits int32+ wantLen int+ }{+ {"endless one-per-page", 500, func(int) int { return 1 }, true, maxLocalePages, 0},+ {"absurd total", 1_000_000, func(int) int { return 1 }, true, 1, 0},+ {"oversized page", 1, func(int) int { return 501 }, true, 1, 0},+ {"exactly the cap", 500, func(int) int { return 100 }, false, 5, 500},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ var hits int32+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ page := int(atomic.AddInt32(&hits, 1))+ type loc struct {+ Code string `json:"code"`+ }+ out := struct {+ Locales []loc `json:"locales"`+ Total int `json:"total"`+ }{Total: tc.total}+ for i := 0; i < tc.perPage(page); i++ {+ out.Locales = append(out.Locales, loc{fmt.Sprintf("l%d-%d", page, i)})+ }+ w.Header().Set("Cache-Control", "max-age=0")+ _ = json.NewEncoder(w).Encode(out)+ }))+ defer srv.Close()++ codes, err := New(srv.URL, "test").Locales(context.Background())+ if (err != nil) != tc.wantErr {+ t.Fatalf("err = %v, wantErr %v", err, tc.wantErr)+ }+ if len(codes) != tc.wantLen {+ t.Errorf("codes = %d, want %d", len(codes), tc.wantLen)+ }+ if got := atomic.LoadInt32(&hits); got != tc.wantHits {+ t.Errorf("hits = %d, want %d", got, tc.wantHits)+ }+ })+ }+}diff --git a/backend/stores/exercise.go b/backend/stores/exercise.goindex 56f678a..62b665a 100644--- a/backend/stores/exercise.go+++ b/backend/stores/exercise.go@@ -559,10 +559,14 @@ func (s *ExerciseStore) RefreshCached(ctx context.Context) (int, error) {//// It stops at the first failed batch and reports how many rows it did apply: the// rows written so far are valid, and the caller decides whether a partial pass counts.+//+// It reads through Revalidating, because a re-read that a cache can answer re-applies+// what this instance already holds.func (s *ExerciseStore) refresh(ctx context.Context, locale string) (int, error) {if s.catalog == nil {return 0, ErrNoCatalog}+ cat := s.catalog.Revalidating()held, err := s.cachedOedbIDs()if err != nil {@@ -573,7 +577,7 @@ func (s *ExerciseStore) refresh(ctx context.Context, locale string) (int, error)}if servedInEnglish(locale) {- items, err := s.catalog.Export(ctx)+ items, err := cat.Export(ctx)if err != nil {return 0, err}@@ -596,7 +600,7 @@ func (s *ExerciseStore) refresh(ctx context.Context, locale string) (int, error)done := 0for start := 0; start < len(ids); start += oedb.MaxPerPage {end := min(start+oedb.MaxPerPage, len(ids))- res, err := s.catalog.List(ctx, oedb.ListParams{IDs: ids[start:end], Locale: locale, PerPage: oedb.MaxPerPage})+ res, err := cat.List(ctx, oedb.ListParams{IDs: ids[start:end], Locale: locale, PerPage: oedb.MaxPerPage})if err != nil {return done, err}diff --git a/backend/stores/exercise_locale_sync.go b/backend/stores/exercise_locale_sync.goindex 156e38a..287bec1 100644--- a/backend/stores/exercise_locale_sync.go+++ b/backend/stores/exercise_locale_sync.go@@ -17,8 +17,9 @@ const CatalogSyncInterval = 24 * time.Hour// a pass comes back clean; CatalogSyncInterval still caps every wait.var catalogRetryBackoff = []time.Duration{time.Minute, 5 * time.Minute, 30 * time.Minute}-// localeSyncTimeout bounds one language, so a hung upstream cannot stall the loop.-const localeSyncTimeout = 2 * time.Minute+// localeSyncTimeout bounds locale discovery and each language, so a hung upstream cannot+// stall the loop. A var so a test can shrink it.+var localeSyncTimeout = 2 * time.Minute// RunCatalogSync re-reads every held exercise in every language the catalog offers, once// at start and then every interval, sooner while a pass is failing. No request triggers it@@ -53,7 +54,9 @@ func retryDelay(failures int, every time.Duration) time.Duration {}func (s *ExerciseStore) syncCatalog(ctx context.Context) error {- codes, err := s.catalog.Locales(ctx)+ dctx, cancel := context.WithTimeout(ctx, localeSyncTimeout)+ codes, err := s.catalog.Revalidating().Locales(dctx)+ cancel()if err != nil {return err}diff --git a/backend/stores/exercise_locale_sync_test.go b/backend/stores/exercise_locale_sync_test.goindex fb48cd9..dfcde40 100644--- a/backend/stores/exercise_locale_sync_test.go+++ b/backend/stores/exercise_locale_sync_test.go@@ -3,6 +3,7 @@ package storesimport ("context""encoding/json"+ "errors""fmt""net/http""net/http/httptest"@@ -14,7 +15,11 @@ import ("github.com/Cawlumm/lyftr-backend/oedb")-type catalogCall struct{ path, locale, ids string }+type catalogCall struct {+ path, locale, ids string+ inm string+ notModified bool+}// localeCatalog is a fake oedb routed by path. It answers the locale list, the English// export of every held id and by-id lists in the requested locale, and records each call.@@ -27,6 +32,11 @@ type localeCatalog struct {held intfailPath map[string]boolfailLocale map[string]bool++ // cacheControl is sent on every response; empty means max-age=0.+ cacheControl string+ // rev changes every body and the ETag; 0 keeps names as they were.+ rev int}func newLocaleCatalog(t *testing.T, held int, locales ...string) *localeCatalog {@@ -35,16 +45,33 @@ func newLocaleCatalog(t *testing.T, held int, locales ...string) *localeCatalogc.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {q := r.URL.Query()c.mu.Lock()- c.calls = append(c.calls, catalogCall{r.URL.Path, q.Get("locale"), q.Get("ids")})failed := c.failPath[r.URL.Path] || c.failLocale[q.Get("locale")] && r.URL.Path == "/api/v1/exercises"locs := append([]string(nil), c.locales...)+ rev := c.rev+ cc := c.cacheControl+ etag := fmt.Sprintf("\"r%d\"", rev)+ inm := r.Header.Get("If-None-Match")+ notModified := !failed && inm == etag+ c.calls = append(c.calls, catalogCall{path: r.URL.Path, locale: q.Get("locale"), ids: q.Get("ids"), inm: inm, notModified: notModified})c.mu.Unlock()// Without this oedb.Client serves a repeat from memory and counts lie.- w.Header().Set("Cache-Control", "max-age=0")+ if cc == "" {+ cc = "max-age=0"+ }+ w.Header().Set("Cache-Control", cc)if failed {http.Error(w, "down", http.StatusInternalServerError)return}+ w.Header().Set("ETag", etag)+ if notModified {+ w.WriteHeader(http.StatusNotModified)+ return+ }+ suffix := ""+ if rev > 0 {+ suffix = fmt.Sprintf(" r%d", rev)+ }switch r.URL.Path {case "/api/v1/meta/locales":type loc struct {@@ -61,14 +88,14 @@ func newLocaleCatalog(t *testing.T, held int, locales ...string) *localeCatalogcase "/api/v1/exercises/export":var items []oedb.Exercisefor i := 0; i < c.held; i++ {- items = append(items, catalogExercise(fmt.Sprintf("uuid-%03d", i), fmt.Sprintf("Exercise %03d", i)))+ items = append(items, catalogExercise(fmt.Sprintf("uuid-%03d", i), fmt.Sprintf("Exercise %03d", i)+suffix))}_ = json.NewEncoder(w).Encode(oedb.ListResult{Exercises: items, Total: len(items)})case "/api/v1/exercises":loc := q.Get("locale")var items []oedb.Exercisefor _, id := range strings.Split(q.Get("ids"), ",") {- items = append(items, served(id, loc+" "+id, loc, []string{"Uno.", "Dos."}, loc))+ items = append(items, served(id, loc+" "+id+suffix, loc, []string{"Uno.", "Dos."}, loc))}_ = json.NewEncoder(w).Encode(oedb.ListResult{Exercises: items, Total: len(items)})default:@@ -99,6 +126,24 @@ func (c *localeCatalog) listCount(locale string) int { return c.count("/api/v1/efunc (c *localeCatalog) localeRequests() int { return c.count("/api/v1/meta/locales", "") }+func (c *localeCatalog) setCacheControl(v string) {+ c.mu.Lock()+ defer c.mu.Unlock()+ c.cacheControl = v+}++func (c *localeCatalog) setRev(n int) {+ c.mu.Lock()+ defer c.mu.Unlock()+ c.rev = n+}++func (c *localeCatalog) setLocales(l ...string) {+ c.mu.Lock()+ defer c.mu.Unlock()+ c.locales = l+}+func (c *localeCatalog) setFail(path string, locale string) {c.mu.Lock()defer c.mu.Unlock()@@ -371,3 +416,90 @@ func TestRunCatalogSync_BacksOffToTheFullIntervalOnceAPassIsClean(t *testing.T)t.Errorf("locale requests = %d after a clean pass, want 1: a clean pass must wait the full interval", got)}}++const slowCacheControl = "public, max-age=300, stale-while-revalidate=86400"++func TestSyncCatalog_SecondPassAppliesUpstreamChangesInsideTheCacheWindow(t *testing.T) {+ s, cat := syncStore(t, 3, "en", "es")+ cat.setCacheControl(slowCacheControl)++ if err := s.syncCatalog(t.Context()); err != nil {+ t.Fatal(err)+ }+ cat.setRev(1)+ cat.setLocales("en", "es", "pt")+ if err := s.syncCatalog(t.Context()); err != nil {+ t.Fatal(err)+ }+ id := localID(t, s, "uuid-001")+ if got := mustGet(t, s, id, "es").Name; got != "es uuid-001 r1" {+ t.Errorf("Get(es).Name = %q, want the second pass's name", got)+ }+ if got := mustGet(t, s, id, "en").Name; got != "Exercise 001 r1" {+ t.Errorf("Get(en).Name = %q, want the second pass's name", got)+ }+ if got := countRows(t, s.db, `SELECT COUNT(*) FROM exercise_translations WHERE locale = 'pt'`); got != 3 {+ t.Errorf("pt translations = %d, want 3 from the newly listed locale", got)+ }+}++func TestSyncCatalog_UnchangedUpstreamIsRevalidatedWithout304Bodies(t *testing.T) {+ s, cat := syncStore(t, 3, "en", "es")+ cat.setCacheControl(slowCacheControl)++ if err := s.syncCatalog(t.Context()); err != nil {+ t.Fatal(err)+ }+ if _, err := s.db.Exec(`UPDATE exercise_translations SET name = 'tampered' WHERE locale = 'es'`); err != nil {+ t.Fatal(err)+ }+ if _, err := s.db.Exec(`UPDATE exercises SET name = 'tampered ' || oedb_id`); err != nil {+ t.Fatal(err)+ }+ before := len(cat.snapshot())+ if err := s.syncCatalog(t.Context()); err != nil {+ t.Fatal(err)+ }+ second := cat.snapshot()[before:]+ if len(second) == 0 {+ t.Fatal("second pass made no requests: it was served from cache")+ }+ for _, c := range second {+ if c.inm != `"r0"` || !c.notModified {+ t.Errorf("call %+v: want a conditional GET answered 304", c)+ }+ }+ id := localID(t, s, "uuid-001")+ if got := mustGet(t, s, id, "es").Name; got != "es uuid-001" {+ t.Errorf("Get(es).Name = %q, want it restored from the cached body", got)+ }+ if got := mustGet(t, s, id, "en").Name; got != "Exercise 001" {+ t.Errorf("Get(en).Name = %q, want it restored from the cached body", got)+ }+}++func TestSyncCatalog_LocaleDiscoveryHasADeadline(t *testing.T) {+ old := localeSyncTimeout+ localeSyncTimeout = 50 * time.Millisecond+ t.Cleanup(func() { localeSyncTimeout = old })++ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ <-r.Context().Done()+ }))+ t.Cleanup(srv.Close)+ s := NewExerciseStore(testDB(t))+ s.UseCatalog(oedb.New(srv.URL, "test"))+ holdEnglish(t, s, 2)++ start := time.Now()+ err := s.syncCatalog(t.Context())+ if !errors.Is(err, context.DeadlineExceeded) {+ t.Fatalf("err = %v, want context.DeadlineExceeded", err)+ }+ if time.Since(start) > 2*time.Second {+ t.Errorf("discovery took %s, want it cut off by the deadline", time.Since(start))+ }+ if got := countRows(t, s.db, `SELECT COUNT(*) FROM exercise_translations WHERE locale <> 'en'`); got != 0 {+ t.Errorf("non-English translations = %d, want 0", got)+ }+}