Skip to content

armgit / lyftr

Public Imported from GitHub
0

Self-hosted fitness tracker. Log workouts, track nutrition, and monitor progress. Docker deployment, mobile-friendly.

Make the catalog sync read fresh upstream data (#206)

Cartera951439f70c3
diff --git a/backend/oedb/client.go b/backend/oedb/client.go
index 678db01..26f95b9 100644
--- a/backend/oedb/client.go
+++ b/backend/oedb/client.go
@@ -42,8 +42,12 @@ type Client struct {
userAgent string
http *http.Client
- mu sync.Mutex
+ // mu is a pointer so a Revalidating view shares the lock along with the map it guards.
+ mu *sync.Mutex
cache map[string]*entry
+
+ // alwaysRevalidate: never answer from cache; every read is a conditional GET.
+ alwaysRevalidate bool
}
// entry is one cached response body plus the metadata needed to revalidate it.
@@ -77,6 +81,7 @@ func New(baseURL, version string) *Client {
// see the context each caller passes in.
http: &http.Client{Timeout: 90 * time.Second},
cache: map[string]*entry{},
+ mu: &sync.Mutex{},
}
}
@@ -254,17 +259,21 @@ func (c *Client) getJSON(ctx context.Context, path string, dst any) error {
// spends most of its life in, and serving through it is what keeps a user's
// search off the critical path of an upstream round trip.
// - beyond both: fetch synchronously.
+//
+// A Revalidating view skips the first two states and always takes the third,
+// conditionally.
func (c *Client) get(ctx context.Context, path string) ([]byte, error) {
now := time.Now()
c.mu.Lock()
e, ok := c.cache[path]
- if ok && now.Before(e.freshFor) {
+ cached := ok && !c.alwaysRevalidate
+ if cached && now.Before(e.freshFor) {
body := e.body
c.mu.Unlock()
return body, nil
}
- if ok && now.Before(e.staleFor) && len(e.body) > 0 {
+ if cached && now.Before(e.staleFor) && len(e.body) > 0 {
body := e.body
if !e.refreshing {
e.refreshing = true
@@ -282,6 +291,17 @@ func (c *Client) get(ctx context.Context, path string) ([]byte, error) {
return c.fetch(ctx, path, etag)
}
+// Revalidating returns a view of c that never answers from cache: every read is a
+// conditional GET with the stored ETag, and a 304 reuses the cached body. It shares
+// c's cache, so what it learns also refreshes what interactive reads are served.
+// The catalog sync reads through it; served from cache, each pass would apply what
+// the previous pass fetched, a whole interval late.
+func (c *Client) Revalidating() *Client {
+ v := *c
+ v.alwaysRevalidate = true
+ return &v
+}
+
// revalidate refreshes a stale entry off the request path. It deliberately uses
// its own timeout-bounded context rather than the caller's: the caller has
// already been served, and cancelling the refresh when that request finishes
@@ -442,11 +462,17 @@ func cacheWindow(header string) (fresh, stale time.Duration) {
return fresh, stale
}
+// maxLocalePages bounds locale discovery. oedb's locale list is a curated taxonomy
+// that fits on one page; a list still going after this many is a broken or hostile
+// upstream, and every listed locale costs the sync a pass of its own.
+const maxLocalePages = 5
+
// Locales lists every locale the catalog holds translated content in; oedb always
-// includes en.
+// includes en. The list is bounded, and a list that exceeds it is an error rather
+// than a partial answer.
func (c *Client) Locales(ctx context.Context) ([]string, error) {
var codes []string
- for page := 1; ; page++ {
+ for page := 1; page <= maxLocalePages; page++ {
var out struct {
Locales []struct {
Code string `json:"code"`
@@ -460,10 +486,14 @@ func (c *Client) Locales(ctx context.Context) ([]string, error) {
for _, l := range out.Locales {
codes = append(codes, l.Code)
}
+ if out.Total > maxLocalePages*MaxPerPage || len(codes) > maxLocalePages*MaxPerPage {
+ return nil, fmt.Errorf("oedb: locale list exceeds %d entries", maxLocalePages*MaxPerPage)
+ }
if len(out.Locales) == 0 || len(codes) >= out.Total {
return codes, nil
}
}
+ return nil, fmt.Errorf("oedb: locale list did not end within %d pages", maxLocalePages)
}
// Export fetches the entire published catalog in one request.
diff --git a/backend/oedb/client_test.go b/backend/oedb/client_test.go
index 8aeada0..a4ac873 100644
--- a/backend/oedb/client_test.go
+++ b/backend/oedb/client_test.go
@@ -455,3 +455,142 @@ func TestLocales_PropagatesErrors(t *testing.T) {
t.Errorf("404: err = %v, want ErrNotFound", err)
}
}
+
+func TestRevalidating_SendsAConditionalGetEvenWhenFresh(t *testing.T) {
+ var hits, bodies int32
+ var secondINM atomic.Value
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ n := atomic.AddInt32(&hits, 1)
+ w.Header().Set("Cache-Control", "public, max-age=300, stale-while-revalidate=86400")
+ w.Header().Set("ETag", `"v1"`)
+ if n == 2 {
+ secondINM.Store(r.Header.Get("If-None-Match"))
+ }
+ if r.Header.Get("If-None-Match") == `"v1"` {
+ w.WriteHeader(http.StatusNotModified)
+ return
+ }
+ atomic.AddInt32(&bodies, 1)
+ fmt.Fprint(w, listBody("Bench Press"))
+ }))
+ defer srv.Close()
+
+ v := New(srv.URL, "test").Revalidating()
+ for i := 0; i < 2; i++ {
+ res, err := v.List(context.Background(), ListParams{})
+ if err != nil {
+ t.Fatalf("list %d: %v", i, err)
+ }
+ if len(res.Exercises) != 1 {
+ t.Fatalf("list %d: %d exercises, want 1", i, len(res.Exercises))
+ }
+ }
+ if got := atomic.LoadInt32(&hits); got != 2 {
+ t.Errorf("hits = %d, want 2: a fresh entry must still be revalidated", got)
+ }
+ if got, _ := secondINM.Load().(string); got != `"v1"` {
+ t.Errorf("second If-None-Match = %q, want \"v1\"", got)
+ }
+ if got := atomic.LoadInt32(&bodies); got != 1 {
+ t.Errorf("full bodies served = %d, want 1", got)
+ }
+}
+
+func TestRevalidating_SharesTheCacheButInteractiveReadsStillServeStale(t *testing.T) {
+ var hits int32
+ release := make(chan struct{})
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ n := atomic.AddInt32(&hits, 1)
+ if n >= 3 {
+ <-release
+ }
+ v := "v1"
+ if n >= 2 {
+ v = "v2"
+ }
+ w.Header().Set("Cache-Control", "public, max-age=0, stale-while-revalidate=86400")
+ w.Header().Set("ETag", `"`+v+`"`)
+ fmt.Fprint(w, listBody(v))
+ }))
+ defer srv.Close()
+ defer close(release)
+
+ c := New(srv.URL, "test")
+ first, err := c.List(context.Background(), ListParams{})
+ if err != nil || first.Exercises[0].Name() != "v1" {
+ t.Fatalf("prime: %v %+v", err, first)
+ }
+ second, err := c.Revalidating().List(context.Background(), ListParams{})
+ if err != nil || second.Exercises[0].Name() != "v2" {
+ t.Fatalf("revalidating read must return the new body synchronously: %v %+v", err, second)
+ }
+
+ done := make(chan string, 1)
+ go func() {
+ res, err := c.List(context.Background(), ListParams{})
+ if err != nil {
+ done <- err.Error()
+ return
+ }
+ done <- res.Exercises[0].Name()
+ }()
+ select {
+ case got := <-done:
+ if got != "v2" {
+ t.Errorf("interactive read = %q, want the view's v2 from the shared cache", got)
+ }
+ case <-time.After(2 * time.Second):
+ t.Fatal("interactive read waited on the network instead of serving stale")
+ }
+ if got := atomic.LoadInt32(&hits); got > 3 {
+ t.Errorf("hits = %d, want at most 2 synchronous plus 1 background", got)
+ }
+}
+
+func TestLocales_BoundedAgainstHostileLists(t *testing.T) {
+ cases := []struct {
+ name string
+ total int
+ perPage func(page int) int
+ wantErr bool
+ wantHits int32
+ wantLen int
+ }{
+ {"endless one-per-page", 500, func(int) int { return 1 }, true, maxLocalePages, 0},
+ {"absurd total", 1_000_000, func(int) int { return 1 }, true, 1, 0},
+ {"oversized page", 1, func(int) int { return 501 }, true, 1, 0},
+ {"exactly the cap", 500, func(int) int { return 100 }, false, 5, 500},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ var hits int32
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ page := int(atomic.AddInt32(&hits, 1))
+ type loc struct {
+ Code string `json:"code"`
+ }
+ out := struct {
+ Locales []loc `json:"locales"`
+ Total int `json:"total"`
+ }{Total: tc.total}
+ for i := 0; i < tc.perPage(page); i++ {
+ out.Locales = append(out.Locales, loc{fmt.Sprintf("l%d-%d", page, i)})
+ }
+ w.Header().Set("Cache-Control", "max-age=0")
+ _ = json.NewEncoder(w).Encode(out)
+ }))
+ defer srv.Close()
+
+ codes, err := New(srv.URL, "test").Locales(context.Background())
+ if (err != nil) != tc.wantErr {
+ t.Fatalf("err = %v, wantErr %v", err, tc.wantErr)
+ }
+ if len(codes) != tc.wantLen {
+ t.Errorf("codes = %d, want %d", len(codes), tc.wantLen)
+ }
+ if got := atomic.LoadInt32(&hits); got != tc.wantHits {
+ t.Errorf("hits = %d, want %d", got, tc.wantHits)
+ }
+ })
+ }
+}
diff --git a/backend/stores/exercise.go b/backend/stores/exercise.go
index 56f678a..62b665a 100644
--- a/backend/stores/exercise.go
+++ b/backend/stores/exercise.go
@@ -559,10 +559,14 @@ func (s *ExerciseStore) RefreshCached(ctx context.Context) (int, error) {
//
// It stops at the first failed batch and reports how many rows it did apply: the
// rows written so far are valid, and the caller decides whether a partial pass counts.
+//
+// It reads through Revalidating, because a re-read that a cache can answer re-applies
+// what this instance already holds.
func (s *ExerciseStore) refresh(ctx context.Context, locale string) (int, error) {
if s.catalog == nil {
return 0, ErrNoCatalog
}
+ cat := s.catalog.Revalidating()
held, err := s.cachedOedbIDs()
if err != nil {
@@ -573,7 +577,7 @@ func (s *ExerciseStore) refresh(ctx context.Context, locale string) (int, error)
}
if servedInEnglish(locale) {
- items, err := s.catalog.Export(ctx)
+ items, err := cat.Export(ctx)
if err != nil {
return 0, err
}
@@ -596,7 +600,7 @@ func (s *ExerciseStore) refresh(ctx context.Context, locale string) (int, error)
done := 0
for start := 0; start < len(ids); start += oedb.MaxPerPage {
end := min(start+oedb.MaxPerPage, len(ids))
- res, err := s.catalog.List(ctx, oedb.ListParams{IDs: ids[start:end], Locale: locale, PerPage: oedb.MaxPerPage})
+ res, err := cat.List(ctx, oedb.ListParams{IDs: ids[start:end], Locale: locale, PerPage: oedb.MaxPerPage})
if err != nil {
return done, err
}
diff --git a/backend/stores/exercise_locale_sync.go b/backend/stores/exercise_locale_sync.go
index 156e38a..287bec1 100644
--- a/backend/stores/exercise_locale_sync.go
+++ b/backend/stores/exercise_locale_sync.go
@@ -17,8 +17,9 @@ const CatalogSyncInterval = 24 * time.Hour
// a pass comes back clean; CatalogSyncInterval still caps every wait.
var catalogRetryBackoff = []time.Duration{time.Minute, 5 * time.Minute, 30 * time.Minute}
-// localeSyncTimeout bounds one language, so a hung upstream cannot stall the loop.
-const localeSyncTimeout = 2 * time.Minute
+// localeSyncTimeout bounds locale discovery and each language, so a hung upstream cannot
+// stall the loop. A var so a test can shrink it.
+var localeSyncTimeout = 2 * time.Minute
// RunCatalogSync re-reads every held exercise in every language the catalog offers, once
// at start and then every interval, sooner while a pass is failing. No request triggers it
@@ -53,7 +54,9 @@ func retryDelay(failures int, every time.Duration) time.Duration {
}
func (s *ExerciseStore) syncCatalog(ctx context.Context) error {
- codes, err := s.catalog.Locales(ctx)
+ dctx, cancel := context.WithTimeout(ctx, localeSyncTimeout)
+ codes, err := s.catalog.Revalidating().Locales(dctx)
+ cancel()
if err != nil {
return err
}
diff --git a/backend/stores/exercise_locale_sync_test.go b/backend/stores/exercise_locale_sync_test.go
index fb48cd9..dfcde40 100644
--- a/backend/stores/exercise_locale_sync_test.go
+++ b/backend/stores/exercise_locale_sync_test.go
@@ -3,6 +3,7 @@ package stores
import (
"context"
"encoding/json"
+ "errors"
"fmt"
"net/http"
"net/http/httptest"
@@ -14,7 +15,11 @@ import (
"github.com/Cawlumm/lyftr-backend/oedb"
)
-type catalogCall struct{ path, locale, ids string }
+type catalogCall struct {
+ path, locale, ids string
+ inm string
+ notModified bool
+}
// localeCatalog is a fake oedb routed by path. It answers the locale list, the English
// export of every held id and by-id lists in the requested locale, and records each call.
@@ -27,6 +32,11 @@ type localeCatalog struct {
held int
failPath map[string]bool
failLocale map[string]bool
+
+ // cacheControl is sent on every response; empty means max-age=0.
+ cacheControl string
+ // rev changes every body and the ETag; 0 keeps names as they were.
+ rev int
}
func newLocaleCatalog(t *testing.T, held int, locales ...string) *localeCatalog {
@@ -35,16 +45,33 @@ func newLocaleCatalog(t *testing.T, held int, locales ...string) *localeCatalog
c.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
c.mu.Lock()
- c.calls = append(c.calls, catalogCall{r.URL.Path, q.Get("locale"), q.Get("ids")})
failed := c.failPath[r.URL.Path] || c.failLocale[q.Get("locale")] && r.URL.Path == "/api/v1/exercises"
locs := append([]string(nil), c.locales...)
+ rev := c.rev
+ cc := c.cacheControl
+ etag := fmt.Sprintf("\"r%d\"", rev)
+ inm := r.Header.Get("If-None-Match")
+ notModified := !failed && inm == etag
+ c.calls = append(c.calls, catalogCall{path: r.URL.Path, locale: q.Get("locale"), ids: q.Get("ids"), inm: inm, notModified: notModified})
c.mu.Unlock()
// Without this oedb.Client serves a repeat from memory and counts lie.
- w.Header().Set("Cache-Control", "max-age=0")
+ if cc == "" {
+ cc = "max-age=0"
+ }
+ w.Header().Set("Cache-Control", cc)
if failed {
http.Error(w, "down", http.StatusInternalServerError)
return
}
+ w.Header().Set("ETag", etag)
+ if notModified {
+ w.WriteHeader(http.StatusNotModified)
+ return
+ }
+ suffix := ""
+ if rev > 0 {
+ suffix = fmt.Sprintf(" r%d", rev)
+ }
switch r.URL.Path {
case "/api/v1/meta/locales":
type loc struct {
@@ -61,14 +88,14 @@ func newLocaleCatalog(t *testing.T, held int, locales ...string) *localeCatalog
case "/api/v1/exercises/export":
var items []oedb.Exercise
for i := 0; i < c.held; i++ {
- items = append(items, catalogExercise(fmt.Sprintf("uuid-%03d", i), fmt.Sprintf("Exercise %03d", i)))
+ items = append(items, catalogExercise(fmt.Sprintf("uuid-%03d", i), fmt.Sprintf("Exercise %03d", i)+suffix))
}
_ = json.NewEncoder(w).Encode(oedb.ListResult{Exercises: items, Total: len(items)})
case "/api/v1/exercises":
loc := q.Get("locale")
var items []oedb.Exercise
for _, id := range strings.Split(q.Get("ids"), ",") {
- items = append(items, served(id, loc+" "+id, loc, []string{"Uno.", "Dos."}, loc))
+ items = append(items, served(id, loc+" "+id+suffix, loc, []string{"Uno.", "Dos."}, loc))
}
_ = json.NewEncoder(w).Encode(oedb.ListResult{Exercises: items, Total: len(items)})
default:
@@ -99,6 +126,24 @@ func (c *localeCatalog) listCount(locale string) int { return c.count("/api/v1/e
func (c *localeCatalog) localeRequests() int { return c.count("/api/v1/meta/locales", "") }
+func (c *localeCatalog) setCacheControl(v string) {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ c.cacheControl = v
+}
+
+func (c *localeCatalog) setRev(n int) {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ c.rev = n
+}
+
+func (c *localeCatalog) setLocales(l ...string) {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ c.locales = l
+}
+
func (c *localeCatalog) setFail(path string, locale string) {
c.mu.Lock()
defer c.mu.Unlock()
@@ -371,3 +416,90 @@ func TestRunCatalogSync_BacksOffToTheFullIntervalOnceAPassIsClean(t *testing.T)
t.Errorf("locale requests = %d after a clean pass, want 1: a clean pass must wait the full interval", got)
}
}
+
+const slowCacheControl = "public, max-age=300, stale-while-revalidate=86400"
+
+func TestSyncCatalog_SecondPassAppliesUpstreamChangesInsideTheCacheWindow(t *testing.T) {
+ s, cat := syncStore(t, 3, "en", "es")
+ cat.setCacheControl(slowCacheControl)
+
+ if err := s.syncCatalog(t.Context()); err != nil {
+ t.Fatal(err)
+ }
+ cat.setRev(1)
+ cat.setLocales("en", "es", "pt")
+ if err := s.syncCatalog(t.Context()); err != nil {
+ t.Fatal(err)
+ }
+ id := localID(t, s, "uuid-001")
+ if got := mustGet(t, s, id, "es").Name; got != "es uuid-001 r1" {
+ t.Errorf("Get(es).Name = %q, want the second pass's name", got)
+ }
+ if got := mustGet(t, s, id, "en").Name; got != "Exercise 001 r1" {
+ t.Errorf("Get(en).Name = %q, want the second pass's name", got)
+ }
+ if got := countRows(t, s.db, `SELECT COUNT(*) FROM exercise_translations WHERE locale = 'pt'`); got != 3 {
+ t.Errorf("pt translations = %d, want 3 from the newly listed locale", got)
+ }
+}
+
+func TestSyncCatalog_UnchangedUpstreamIsRevalidatedWithout304Bodies(t *testing.T) {
+ s, cat := syncStore(t, 3, "en", "es")
+ cat.setCacheControl(slowCacheControl)
+
+ if err := s.syncCatalog(t.Context()); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := s.db.Exec(`UPDATE exercise_translations SET name = 'tampered' WHERE locale = 'es'`); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := s.db.Exec(`UPDATE exercises SET name = 'tampered ' || oedb_id`); err != nil {
+ t.Fatal(err)
+ }
+ before := len(cat.snapshot())
+ if err := s.syncCatalog(t.Context()); err != nil {
+ t.Fatal(err)
+ }
+ second := cat.snapshot()[before:]
+ if len(second) == 0 {
+ t.Fatal("second pass made no requests: it was served from cache")
+ }
+ for _, c := range second {
+ if c.inm != `"r0"` || !c.notModified {
+ t.Errorf("call %+v: want a conditional GET answered 304", c)
+ }
+ }
+ id := localID(t, s, "uuid-001")
+ if got := mustGet(t, s, id, "es").Name; got != "es uuid-001" {
+ t.Errorf("Get(es).Name = %q, want it restored from the cached body", got)
+ }
+ if got := mustGet(t, s, id, "en").Name; got != "Exercise 001" {
+ t.Errorf("Get(en).Name = %q, want it restored from the cached body", got)
+ }
+}
+
+func TestSyncCatalog_LocaleDiscoveryHasADeadline(t *testing.T) {
+ old := localeSyncTimeout
+ localeSyncTimeout = 50 * time.Millisecond
+ t.Cleanup(func() { localeSyncTimeout = old })
+
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ <-r.Context().Done()
+ }))
+ t.Cleanup(srv.Close)
+ s := NewExerciseStore(testDB(t))
+ s.UseCatalog(oedb.New(srv.URL, "test"))
+ holdEnglish(t, s, 2)
+
+ start := time.Now()
+ err := s.syncCatalog(t.Context())
+ if !errors.Is(err, context.DeadlineExceeded) {
+ t.Fatalf("err = %v, want context.DeadlineExceeded", err)
+ }
+ if time.Since(start) > 2*time.Second {
+ t.Errorf("discovery took %s, want it cut off by the deadline", time.Since(start))
+ }
+ if got := countRows(t, s.db, `SELECT COUNT(*) FROM exercise_translations WHERE locale <> 'en'`); got != 0 {
+ t.Errorf("non-English translations = %d, want 0", got)
+ }
+}