Skip to content

armgit / lyftr

Public Imported from GitHub
0

Self-hosted fitness tracker. Log workouts, track nutrition, and monitor progress. Docker deployment, mobile-friendly.

Keep the exercise cache filled in every language, on a schedule (#205)

Carterfb72b767cf3b
diff --git a/.env.example b/.env.example
index 250b723..467371a 100644
--- a/.env.example
+++ b/.env.example
@@ -14,7 +14,8 @@ DEMO_MODE=false
# The open-exercise-db instance the exercise catalog is queried from. Lyftr does not
# ship or seed an exercise library: it asks this API as you search, and keeps only the
# rows it has shown. Unset uses the hosted instance; point it at your own to self-host
-# the catalog too.
+# the catalog too. Those rows are re-read once a day, at startup and every 24 hours after,
+# in every language the catalog offers.
# OEDB_BASE_URL=https://oedb-api-o4dqddih5q-uk.a.run.app
# Address the frontend's nginx proxies /api to, over the internal Docker network.
diff --git a/backend/main.go b/backend/main.go
index 28bdd31..1aaef53 100644
--- a/backend/main.go
+++ b/backend/main.go
@@ -1,6 +1,7 @@
package main
import (
+ "context"
"flag"
"fmt"
"log"
@@ -113,9 +114,13 @@ func main() {
s := stores.New(db.DB)
s.Exercise.UseCatalog(oedb.New(config.C.OEDBBaseURL, config.Version()))
- if config.C.DemoMode {
- go seed.DemoData(db.DB, s.Exercise)
- }
+ // The demo's exercises are primed first so the boot pass translates them too.
+ go func() {
+ if config.C.DemoMode {
+ seed.DemoData(db.DB, s.Exercise)
+ }
+ s.Exercise.RunCatalogSync(context.Background(), stores.CatalogSyncInterval)
+ }()
if config.C.Env == "production" {
gin.SetMode(gin.ReleaseMode)
diff --git a/backend/oedb/client.go b/backend/oedb/client.go
index 539553b..678db01 100644
--- a/backend/oedb/client.go
+++ b/backend/oedb/client.go
@@ -2,9 +2,9 @@
// exercise catalog.
//
// Lyftr queries oedb rather than mirroring it. The local exercises table still
-// exists, but as a snapshot of rows a user has actually seen — see
-// stores.ExerciseStore.Materialize — not as a copy of the catalog kept in step by
-// a periodic sync. Search, filtering and ordering are answered upstream, so the
+// exists, but as the rows a user has actually seen — see
+// stores.ExerciseStore.Materialize — which a daily pass keeps current in every
+// catalog language, and not as a copy of the catalog. Search, filtering and ordering are answered upstream, so the
// data Lyftr shows improves when oedb improves, without a redeploy.
//
// The cache here is what makes that affordable. oedb serves list and detail reads
@@ -33,8 +33,8 @@ import (
// self-hosted Lyftr at a self-hosted catalog.
const DefaultBaseURL = "https://oedb-api-o4dqddih5q-uk.a.run.app"
-// maxPerPage is oedb's cap; asking for more is a 422, not a clamp.
-const maxPerPage = 100
+// MaxPerPage is oedb's cap; asking for more is a 422, not a clamp.
+const MaxPerPage = 100
// Client is a cache-aware reader for the oedb HTTP API. Safe for concurrent use.
type Client struct {
@@ -115,10 +115,10 @@ func (p ListParams) query() url.Values {
}
per := p.PerPage
if per <= 0 {
- per = maxPerPage
+ per = MaxPerPage
}
- if per > maxPerPage {
- per = maxPerPage
+ if per > MaxPerPage {
+ per = MaxPerPage
}
v.Set("per_page", strconv.Itoa(per))
return v
@@ -442,6 +442,30 @@ func cacheWindow(header string) (fresh, stale time.Duration) {
return fresh, stale
}
+// Locales lists every locale the catalog holds translated content in; oedb always
+// includes en.
+func (c *Client) Locales(ctx context.Context) ([]string, error) {
+ var codes []string
+ for page := 1; ; page++ {
+ var out struct {
+ Locales []struct {
+ Code string `json:"code"`
+ } `json:"locales"`
+ Total int `json:"total"`
+ }
+ path := "/api/v1/meta/locales?page=" + strconv.Itoa(page) + "&per_page=" + strconv.Itoa(MaxPerPage)
+ if err := c.getJSON(ctx, path, &out); err != nil {
+ return nil, err
+ }
+ for _, l := range out.Locales {
+ codes = append(codes, l.Code)
+ }
+ if len(out.Locales) == 0 || len(codes) >= out.Total {
+ return codes, nil
+ }
+ }
+}
+
// Export fetches the entire published catalog in one request.
//
// oedb caps per_page at 100, so assembling all ~873 exercises through List would
diff --git a/backend/oedb/client_test.go b/backend/oedb/client_test.go
index 6cf6891..8aeada0 100644
--- a/backend/oedb/client_test.go
+++ b/backend/oedb/client_test.go
@@ -404,3 +404,54 @@ func TestTranslation_DecodesServedLocalesAndMachineFlags(t *testing.T) {
t.Errorf("decoded %+v", tr)
}
}
+
+func TestLocales_WalksEveryPage(t *testing.T) {
+ var agents []string
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Query().Get("per_page") != "100" {
+ t.Errorf("per_page = %q, want 100", r.URL.Query().Get("per_page"))
+ }
+ agents = append(agents, r.UserAgent())
+ w.Header().Set("Cache-Control", "max-age=0")
+ if r.URL.Query().Get("page") == "1" {
+ fmt.Fprint(w, `{"locales":[{"code":"en"},{"code":"es"}],"total":3}`)
+ return
+ }
+ fmt.Fprint(w, `{"locales":[{"code":"pt"}],"total":3}`)
+ }))
+ defer srv.Close()
+
+ got, err := New(srv.URL, "test").Locales(context.Background())
+ if err != nil {
+ t.Fatal(err)
+ }
+ if strings.Join(got, ",") != "en,es,pt" {
+ t.Errorf("Locales = %v, want [en es pt]", got)
+ }
+ if len(agents) != 2 {
+ t.Fatalf("requests = %d, want 2", len(agents))
+ }
+ for _, ua := range agents {
+ if !strings.Contains(ua, "test") {
+ t.Errorf("User-Agent %q does not identify the client", ua)
+ }
+ }
+}
+
+func TestLocales_PropagatesErrors(t *testing.T) {
+ status := http.StatusInternalServerError
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ http.Error(w, "x", status)
+ }))
+ defer srv.Close()
+
+ c := New(srv.URL, "test")
+ if _, err := c.Locales(context.Background()); err == nil {
+ t.Error("500: want an error")
+ }
+ status = http.StatusNotFound
+ c = New(srv.URL, "test")
+ if _, err := c.Locales(context.Background()); !errors.Is(err, ErrNotFound) {
+ t.Errorf("404: err = %v, want ErrNotFound", err)
+ }
+}
diff --git a/backend/stores/exercise.go b/backend/stores/exercise.go
index 19a4fc5..56f678a 100644
--- a/backend/stores/exercise.go
+++ b/backend/stores/exercise.go
@@ -7,6 +7,7 @@ import (
"errors"
"fmt"
"log"
+ "sort"
"strings"
"time"
@@ -548,6 +549,17 @@ func (s *ExerciseStore) CachedCount() (int, error) { return s.Count() }
// by id, because oedb has no batch-by-ids endpoint: refreshing 873 rows one at a
// time would be 873 requests against a 60/minute limit, or about a quarter hour.
func (s *ExerciseStore) RefreshCached(ctx context.Context) (int, error) {
+ return s.refresh(ctx, defaultLocale)
+}
+
+// refresh re-reads every held row in one language. English comes from the bulk
+// export; any other language is read by id in oedb's batch size, because the export
+// is English-only and a list call is the only way to ask for a translation. One code
+// path serves every language, so adding a locale needs no new fetching logic.
+//
+// It stops at the first failed batch and reports how many rows it did apply: the
+// rows written so far are valid, and the caller decides whether a partial pass counts.
+func (s *ExerciseStore) refresh(ctx context.Context, locale string) (int, error) {
if s.catalog == nil {
return 0, ErrNoCatalog
}
@@ -560,23 +572,41 @@ func (s *ExerciseStore) RefreshCached(ctx context.Context) (int, error) {
return 0, nil
}
- items, err := s.catalog.Export(ctx)
- if err != nil {
- return 0, err
+ if servedInEnglish(locale) {
+ items, err := s.catalog.Export(ctx)
+ if err != nil {
+ return 0, err
+ }
+ keep := items[:0]
+ for _, it := range items {
+ if held[it.ID] {
+ keep = append(keep, it)
+ }
+ }
+ out, err := s.Materialize(keep, s.catalog.BaseURL(), defaultLocale)
+ return len(out), err
}
- keep := items[:0]
- for _, it := range items {
- if held[it.ID] {
- keep = append(keep, it)
- }
+ ids := make([]string, 0, len(held))
+ for id := range held {
+ ids = append(ids, id)
}
+ sort.Strings(ids)
- out, err := s.Materialize(keep, s.catalog.BaseURL(), defaultLocale)
- if err != nil {
- return 0, err
+ done := 0
+ for start := 0; start < len(ids); start += oedb.MaxPerPage {
+ end := min(start+oedb.MaxPerPage, len(ids))
+ res, err := s.catalog.List(ctx, oedb.ListParams{IDs: ids[start:end], Locale: locale, PerPage: oedb.MaxPerPage})
+ if err != nil {
+ return done, err
+ }
+ out, err := s.Materialize(res.Exercises, s.catalog.BaseURL(), locale)
+ if err != nil {
+ return done, err
+ }
+ done += len(out)
}
- return len(out), nil
+ return done, nil
}
func (s *ExerciseStore) cachedOedbIDs() (map[string]bool, error) {
diff --git a/backend/stores/exercise_locale_sync.go b/backend/stores/exercise_locale_sync.go
new file mode 100644
index 0000000..156e38a
--- /dev/null
+++ b/backend/stores/exercise_locale_sync.go
@@ -0,0 +1,87 @@
+package stores
+
+import (
+ "context"
+ "fmt"
+ "log"
+ "sort"
+ "time"
+)
+
+// CatalogSyncInterval is how long an upstream correction or a new translation takes to
+// reach readers.
+const CatalogSyncInterval = 24 * time.Hour
+
+// catalogRetryBackoff spaces out retries after a failed pass, so a blip at boot costs
+// minutes of stale text rather than a day. Past the last step it repeats that step until
+// a pass comes back clean; CatalogSyncInterval still caps every wait.
+var catalogRetryBackoff = []time.Duration{time.Minute, 5 * time.Minute, 30 * time.Minute}
+
+// localeSyncTimeout bounds one language, so a hung upstream cannot stall the loop.
+const localeSyncTimeout = 2 * time.Minute
+
+// RunCatalogSync re-reads every held exercise in every language the catalog offers, once
+// at start and then every interval, sooner while a pass is failing. No request triggers it
+// or waits on it; it blocks until ctx is done, so callers run it in a goroutine.
+func (s *ExerciseStore) RunCatalogSync(ctx context.Context, every time.Duration) {
+ if s.catalog == nil {
+ return
+ }
+ failures := 0
+ for {
+ wait := every
+ if err := s.syncCatalog(ctx); err != nil {
+ wait = retryDelay(failures, every)
+ failures++
+ log.Printf("exercises: catalog sync incomplete, will try again in %s: %v", wait, err)
+ } else {
+ failures = 0
+ }
+ select {
+ case <-ctx.Done():
+ return
+ case <-time.After(wait):
+ }
+ }
+}
+
+// retryDelay is the wait after the failures-th consecutive failed pass, never longer than
+// the regular interval.
+func retryDelay(failures int, every time.Duration) time.Duration {
+ d := catalogRetryBackoff[min(failures, len(catalogRetryBackoff)-1)]
+ return min(d, every)
+}
+
+func (s *ExerciseStore) syncCatalog(ctx context.Context) error {
+ codes, err := s.catalog.Locales(ctx)
+ if err != nil {
+ return err
+ }
+ seen := map[string]bool{}
+ var locales []string
+ for _, c := range codes {
+ if l := NormalizeLocale(c); !seen[l] {
+ seen[l] = true
+ locales = append(locales, l)
+ }
+ }
+ // replaceTranslation for es-MX deletes the es row it falls back over, so every
+ // regional tag must be synced before its base language.
+ sort.Sort(sort.Reverse(sort.StringSlice(locales)))
+ failed := 0
+ for _, l := range locales {
+ lctx, cancel := context.WithTimeout(ctx, localeSyncTimeout)
+ n, err := s.refresh(lctx, l)
+ cancel()
+ if err != nil {
+ failed++
+ log.Printf("exercises: catalog sync for %q failed after %d rows: %.200v", l, n, err)
+ continue
+ }
+ log.Printf("exercises: refreshed %d cached exercises in %q", n, l)
+ }
+ if failed > 0 {
+ return fmt.Errorf("%d of %d languages did not sync", failed, len(locales))
+ }
+ return nil
+}
diff --git a/backend/stores/exercise_locale_sync_test.go b/backend/stores/exercise_locale_sync_test.go
new file mode 100644
index 0000000..fb48cd9
--- /dev/null
+++ b/backend/stores/exercise_locale_sync_test.go
@@ -0,0 +1,373 @@
+package stores
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/Cawlumm/lyftr-backend/oedb"
+)
+
+type catalogCall struct{ path, locale, ids string }
+
+// localeCatalog is a fake oedb routed by path. It answers the locale list, the English
+// export of every held id and by-id lists in the requested locale, and records each call.
+type localeCatalog struct {
+ srv *httptest.Server
+
+ mu sync.Mutex
+ calls []catalogCall
+ locales []string
+ held int
+ failPath map[string]bool
+ failLocale map[string]bool
+}
+
+func newLocaleCatalog(t *testing.T, held int, locales ...string) *localeCatalog {
+ t.Helper()
+ c := &localeCatalog{locales: locales, held: held, failPath: map[string]bool{}, failLocale: map[string]bool{}}
+ c.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ q := r.URL.Query()
+ c.mu.Lock()
+ c.calls = append(c.calls, catalogCall{r.URL.Path, q.Get("locale"), q.Get("ids")})
+ failed := c.failPath[r.URL.Path] || c.failLocale[q.Get("locale")] && r.URL.Path == "/api/v1/exercises"
+ locs := append([]string(nil), c.locales...)
+ c.mu.Unlock()
+ // Without this oedb.Client serves a repeat from memory and counts lie.
+ w.Header().Set("Cache-Control", "max-age=0")
+ if failed {
+ http.Error(w, "down", http.StatusInternalServerError)
+ return
+ }
+ switch r.URL.Path {
+ case "/api/v1/meta/locales":
+ type loc struct {
+ Code string `json:"code"`
+ }
+ out := struct {
+ Locales []loc `json:"locales"`
+ Total int `json:"total"`
+ }{Total: len(locs)}
+ for _, l := range locs {
+ out.Locales = append(out.Locales, loc{l})
+ }
+ _ = json.NewEncoder(w).Encode(out)
+ case "/api/v1/exercises/export":
+ var items []oedb.Exercise
+ for i := 0; i < c.held; i++ {
+ items = append(items, catalogExercise(fmt.Sprintf("uuid-%03d", i), fmt.Sprintf("Exercise %03d", i)))
+ }
+ _ = json.NewEncoder(w).Encode(oedb.ListResult{Exercises: items, Total: len(items)})
+ case "/api/v1/exercises":
+ loc := q.Get("locale")
+ var items []oedb.Exercise
+ for _, id := range strings.Split(q.Get("ids"), ",") {
+ items = append(items, served(id, loc+" "+id, loc, []string{"Uno.", "Dos."}, loc))
+ }
+ _ = json.NewEncoder(w).Encode(oedb.ListResult{Exercises: items, Total: len(items)})
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ t.Cleanup(c.srv.Close)
+ return c
+}
+
+func (c *localeCatalog) snapshot() []catalogCall {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ return append([]catalogCall(nil), c.calls...)
+}
+
+func (c *localeCatalog) count(path, locale string) int {
+ n := 0
+ for _, call := range c.snapshot() {
+ if call.path == path && (locale == "" || call.locale == locale) {
+ n++
+ }
+ }
+ return n
+}
+
+func (c *localeCatalog) listCount(locale string) int { return c.count("/api/v1/exercises", locale) }
+
+func (c *localeCatalog) localeRequests() int { return c.count("/api/v1/meta/locales", "") }
+
+func (c *localeCatalog) setFail(path string, locale string) {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ if path != "" {
+ c.failPath[path] = true
+ }
+ if locale != "" {
+ c.failLocale[locale] = true
+ }
+}
+
+func holdEnglish(t *testing.T, s *ExerciseStore, n int) {
+ t.Helper()
+ items := make([]oedb.Exercise, n)
+ for i := range items {
+ items[i] = catalogExercise(fmt.Sprintf("uuid-%03d", i), fmt.Sprintf("Exercise %03d", i))
+ }
+ materialize(t, s, "en", items...)
+}
+
+func syncStore(t *testing.T, held int, locales ...string) (*ExerciseStore, *localeCatalog) {
+ t.Helper()
+ cat := newLocaleCatalog(t, held, locales...)
+ s := NewExerciseStore(testDB(t))
+ s.UseCatalog(oedb.New(cat.srv.URL, "test"))
+ holdEnglish(t, s, held)
+ return s, cat
+}
+
+func localID(t *testing.T, s *ExerciseStore, oedbID string) int64 {
+ t.Helper()
+ var id int64
+ if err := s.db.QueryRow(`SELECT id FROM exercises WHERE oedb_id = ?`, oedbID).Scan(&id); err != nil {
+ t.Fatal(err)
+ }
+ return id
+}
+
+func TestRefresh_FillsEveryHeldRowInBatches(t *testing.T) {
+ s, cat := syncStore(t, 250)
+
+ if _, err := s.refresh(t.Context(), "es"); err != nil {
+ t.Fatal(err)
+ }
+ if got := cat.listCount("es"); got != 3 {
+ t.Errorf("requests = %d, want 3 (250 ids at oedb's page size of 100)", got)
+ }
+ for _, c := range cat.snapshot() {
+ if c.locale != "es" || len(strings.Split(c.ids, ",")) > oedb.MaxPerPage {
+ t.Errorf("request %+v: want locale=es and at most %d ids", c, oedb.MaxPerPage)
+ }
+ }
+ if got := countRows(t, s.db, `SELECT COUNT(*) FROM exercise_translations WHERE locale = 'es'`); got != 250 {
+ t.Errorf("es translations = %d, want 250", got)
+ }
+ id := localID(t, s, "uuid-007")
+ if got := mustGet(t, s, id, "es").Name; got != "es uuid-007" {
+ t.Errorf("Get(es).Name = %q, want the Spanish name", got)
+ }
+ if got := mustGet(t, s, id, "en").Name; got != "Exercise 007" {
+ t.Errorf("Get(en).Name = %q: the English base must be untouched", got)
+ }
+}
+
+func TestSyncCatalog_SyncsEveryLanguageTheCatalogLists(t *testing.T) {
+ s, cat := syncStore(t, 5, "en", "es", "pt")
+
+ if err := s.syncCatalog(t.Context()); err != nil {
+ t.Fatal(err)
+ }
+ if got := cat.count("/api/v1/exercises/export", ""); got != 1 {
+ t.Errorf("export requests = %d, want 1", got)
+ }
+ if cat.listCount("es") != 1 || cat.listCount("pt") != 1 {
+ t.Errorf("list requests es=%d pt=%d, want 1 each", cat.listCount("es"), cat.listCount("pt"))
+ }
+ for _, l := range []string{"es", "pt"} {
+ if got := countRows(t, s.db, `SELECT COUNT(*) FROM exercise_translations WHERE locale = ?`, l); got != 5 {
+ t.Errorf("%s translations = %d, want 5", l, got)
+ }
+ }
+ if got := mustGet(t, s, localID(t, s, "uuid-002"), "pt").Name; got != "pt uuid-002" {
+ t.Errorf("Get(pt).Name = %q, want the Portuguese name", got)
+ }
+}
+
+func TestSyncCatalog_RegionalBeforeBase(t *testing.T) {
+ s, cat := syncStore(t, 3, "en", "es", "es-MX")
+
+ if err := s.syncCatalog(t.Context()); err != nil {
+ t.Fatal(err)
+ }
+ id := localID(t, s, "uuid-001")
+ if got := mustGet(t, s, id, "es").Name; got != "es uuid-001" {
+ t.Errorf("Get(es).Name = %q, want the es name", got)
+ }
+ if got := mustGet(t, s, id, "es-MX").Name; got != "es-MX uuid-001" {
+ t.Errorf("Get(es-MX).Name = %q, want the es-MX name", got)
+ }
+ if got := mustGet(t, s, id, "en").Name; got != "Exercise 001" {
+ t.Errorf("Get(en).Name = %q, want English", got)
+ }
+ var order []string
+ for _, c := range cat.snapshot() {
+ if c.path == "/api/v1/exercises" {
+ order = append(order, c.locale)
+ }
+ }
+ if len(order) < 2 || order[0] != "es-MX" || order[len(order)-1] != "es" {
+ t.Errorf("list order = %v, want es-MX before es", order)
+ }
+}
+
+func TestSyncCatalog_NormalizesAndDedupesDiscoveredCodes(t *testing.T) {
+ s, cat := syncStore(t, 3, "EN", "es", "ES", "bad tag!")
+
+ if err := s.syncCatalog(t.Context()); err != nil {
+ t.Fatal(err)
+ }
+ if got := cat.count("/api/v1/exercises/export", ""); got != 1 {
+ t.Errorf("export requests = %d, want 1", got)
+ }
+ if got := cat.listCount("es"); got != 1 {
+ t.Errorf("es list requests = %d, want 1", got)
+ }
+ for _, c := range cat.snapshot() {
+ if strings.Contains(c.locale, "bad") {
+ t.Errorf("sent locale %q upstream", c.locale)
+ }
+ }
+}
+
+func TestSyncCatalog_DiscoveryFailureSyncsNothing(t *testing.T) {
+ s, cat := syncStore(t, 3, "en", "es")
+ cat.setFail("/api/v1/meta/locales", "")
+
+ if err := s.syncCatalog(t.Context()); err == nil {
+ t.Fatal("want the discovery failure reported")
+ }
+ if got := cat.listCount(""); got != 0 {
+ t.Errorf("list requests = %d, want 0", got)
+ }
+ if got := countRows(t, s.db, `SELECT COUNT(*) FROM exercise_translations WHERE locale <> 'en'`); got != 0 {
+ t.Errorf("non-English translations = %d, want 0", got)
+ }
+ if got := countRows(t, s.db, `SELECT COUNT(*) FROM exercises`); got != 3 {
+ t.Errorf("exercises = %d, want 3 untouched", got)
+ }
+}
+
+func TestSyncCatalog_OneLanguageFailingDoesNotStopTheRest(t *testing.T) {
+ s, cat := syncStore(t, 4, "en", "es", "pt")
+ cat.setFail("", "es")
+
+ if err := s.syncCatalog(t.Context()); err == nil {
+ t.Fatal("want the failed language reported so the loop retries sooner")
+ }
+ if got := countRows(t, s.db, `SELECT COUNT(*) FROM exercise_translations WHERE locale = 'pt'`); got != 4 {
+ t.Errorf("pt translations = %d, want 4", got)
+ }
+ if got := countRows(t, s.db, `SELECT COUNT(*) FROM exercise_translations WHERE locale = 'es'`); got != 0 {
+ t.Errorf("es translations = %d, want 0", got)
+ }
+}
+
+func TestSyncCatalog_EmptyCacheRequestsOnlyLocales(t *testing.T) {
+ s, cat := syncStore(t, 0, "en", "es")
+
+ if err := s.syncCatalog(t.Context()); err != nil {
+ t.Fatal(err)
+ }
+ if got := len(cat.snapshot()); got != 1 || cat.localeRequests() != 1 {
+ t.Errorf("calls = %+v, want only the locales request", cat.snapshot())
+ }
+}
+
+func TestRunCatalogSync_RunsAtStartAndRepeatsUntilCancelled(t *testing.T) {
+ s, cat := syncStore(t, 2, "en")
+
+ ctx, cancel := context.WithCancel(t.Context())
+ defer cancel()
+ done := make(chan struct{})
+ go func() {
+ s.RunCatalogSync(ctx, 10*time.Millisecond)
+ close(done)
+ }()
+ waitFor(t, func() bool { return cat.localeRequests() >= 2 })
+ cancel()
+ select {
+ case <-done:
+ case <-time.After(time.Second):
+ t.Fatal("RunCatalogSync did not return after cancel")
+ }
+}
+
+func TestRunCatalogSync_WithoutCatalogReturnsImmediately(t *testing.T) {
+ s := NewExerciseStore(testDB(t))
+ done := make(chan struct{})
+ go func() {
+ s.RunCatalogSync(t.Context(), time.Hour)
+ close(done)
+ }()
+ select {
+ case <-done:
+ case <-time.After(time.Second):
+ t.Fatal("RunCatalogSync blocked with no catalog")
+ }
+}
+
+func waitFor(t *testing.T, cond func() bool) {
+ t.Helper()
+ deadline := time.Now().Add(3 * time.Second)
+ for !cond() {
+ if time.Now().After(deadline) {
+ t.Fatal("condition not reached in time")
+ }
+ time.Sleep(5 * time.Millisecond)
+ }
+}
+
+func TestRetryDelay_StepsUpThenHoldsAndNeverExceedsTheInterval(t *testing.T) {
+ cases := []struct {
+ failures int
+ every time.Duration
+ want time.Duration
+ }{
+ {0, CatalogSyncInterval, time.Minute},
+ {1, CatalogSyncInterval, 5 * time.Minute},
+ {2, CatalogSyncInterval, 30 * time.Minute},
+ {9, CatalogSyncInterval, 30 * time.Minute},
+ {0, 10 * time.Second, 10 * time.Second},
+ }
+ for _, c := range cases {
+ if got := retryDelay(c.failures, c.every); got != c.want {
+ t.Errorf("retryDelay(%d, %s) = %s, want %s", c.failures, c.every, got, c.want)
+ }
+ }
+}
+
+func TestRunCatalogSync_RetriesSoonAfterAFailedLanguageNotAfterTheFullInterval(t *testing.T) {
+ old := catalogRetryBackoff
+ catalogRetryBackoff = []time.Duration{5 * time.Millisecond}
+ t.Cleanup(func() { catalogRetryBackoff = old })
+
+ s, cat := syncStore(t, 2, "en", "es")
+ cat.setFail("", "es")
+
+ ctx, cancel := context.WithCancel(t.Context())
+ defer cancel()
+ go s.RunCatalogSync(ctx, time.Hour)
+
+ // A one-hour interval would never repeat inside the test; only the backoff can.
+ waitFor(t, func() bool { return cat.listCount("es") >= 3 })
+}
+
+func TestRunCatalogSync_BacksOffToTheFullIntervalOnceAPassIsClean(t *testing.T) {
+ old := catalogRetryBackoff
+ catalogRetryBackoff = []time.Duration{5 * time.Millisecond}
+ t.Cleanup(func() { catalogRetryBackoff = old })
+
+ s, cat := syncStore(t, 2, "en", "es")
+
+ ctx, cancel := context.WithCancel(t.Context())
+ defer cancel()
+ go s.RunCatalogSync(ctx, time.Hour)
+
+ waitFor(t, func() bool { return cat.localeRequests() >= 1 && cat.listCount("es") >= 1 })
+ time.Sleep(100 * time.Millisecond)
+ if got := cat.localeRequests(); got != 1 {
+ t.Errorf("locale requests = %d after a clean pass, want 1: a clean pass must wait the full interval", got)
+ }
+}
diff --git a/docs/API_CHANGES.md b/docs/API_CHANGES.md
index cb4e837..fa1120b 100644
--- a/docs/API_CHANGES.md
+++ b/docs/API_CHANGES.md
@@ -298,3 +298,11 @@ somebody will see the change.
base language and then to English. With no `locale`, or a malformed one, responses are
unchanged: English, status `200`. Local search also matches translated names in that
locale.
+ At startup and every 24 hours after, the instance re-reads every exercise it already holds
+ in every language the catalog lists at oedb's `/api/v1/meta/locales`, English included:
+ English through one export request, every other language with one request per 100
+ exercises. That is how workout and program history follow a language switch and how
+ upstream corrections arrive. No request triggers this pass or waits for it, and it adds no
+ exercises the instance did not already hold. If the catalog cannot be reached, the pass is
+ retried at the next interval and responses are served from what is already held. Nothing
+ changes for an instance with no catalog configured.