Skip to content

armgit / lyftr

Public Imported from GitHub
0

Self-hosted fitness tracker. Log workouts, track nutrition, and monitor progress. Docker deployment, mobile-friendly.

Hide exercises removed upstream and drop languages oedb no longer offers (#214)

Carter0bca03a32aa6
diff --git a/backend/controllers/exercises_test.go b/backend/controllers/exercises_test.go
index 4ac26c7..333b114 100644
--- a/backend/controllers/exercises_test.go
+++ b/backend/controllers/exercises_test.go
@@ -343,3 +343,46 @@ func TestExerciseCacheStatus_afterSuccessfulRefresh(t *testing.T) {
t.Errorf("last_synced_at = %q, want RFC3339 UTC", at)
}
}
+
+// An exercise removed upstream leaves the picker feed but stays a valid reference: the
+// FK is satisfied, so history renders and it can still be logged.
+func TestExercises_RemovedUpstreamHiddenFromListButResolvable(t *testing.T) {
+ setupTestDB(t)
+ uid := createTestUser(t)
+ wid, exID := seedSpanishSquat(t, uid)
+ if _, err := db.DB.Exec(`UPDATE exercises SET oedb_id = 'uuid-1', upstream_missing_at = CURRENT_TIMESTAMP WHERE id = ?`, exID); err != nil {
+ t.Fatal(err)
+ }
+
+ c, w := newContext(uid, http.MethodGet, "/api/v1/exercises?q=squat", nil)
+ th.ListExercises(c)
+ if list := decodeResponse(t, w)["data"].([]any); len(list) != 0 {
+ t.Errorf("list = %v, want the removed exercise left out", list)
+ }
+
+ c, w = newContext(uid, http.MethodGet, fmt.Sprintf("/api/v1/exercises/%d", exID), nil)
+ setParam(c, "id", fmt.Sprintf("%d", exID))
+ th.GetExercise(c)
+ if w.Code != http.StatusOK || decodeResponse(t, w)["data"].(map[string]any)["name"] != "Squat" {
+ t.Errorf("GetExercise = %d %s, want 200 Squat", w.Code, w.Body.String())
+ }
+
+ c, w = newContext(uid, http.MethodGet, fmt.Sprintf("/api/v1/workouts/%d", wid), nil)
+ setParam(c, "id", fmt.Sprintf("%d", wid))
+ th.GetWorkout(c)
+ data := decodeResponse(t, w)["data"].(map[string]any)
+ ex := data["exercises"].([]any)[0].(map[string]any)["exercise"].(map[string]any)
+ if ex["name"] != "Squat" {
+ t.Errorf("workout exercise = %v, want it still joined by name", ex)
+ }
+
+ body := map[string]any{
+ "name": "Again",
+ "exercises": []map[string]any{{"exercise_id": exID, "sets": []map[string]any{{"set_number": 1, "reps": 5, "weight": 100.0}}}},
+ }
+ c, w = newContext(uid, http.MethodPost, "/api/v1/workouts", body)
+ th.CreateWorkout(c)
+ if w.Code != http.StatusCreated {
+ t.Errorf("logging a removed exercise = %d %s, want 201", w.Code, w.Body.String())
+ }
+}
diff --git a/backend/db/migrations.go b/backend/db/migrations.go
index 800f76b..3e8b70a 100644
--- a/backend/db/migrations.go
+++ b/backend/db/migrations.go
@@ -142,6 +142,11 @@ func alterMigrations() {
ensureColumn("exercises", "slug", `ALTER TABLE exercises ADD COLUMN slug TEXT`)
ensureIndex("idx_exercises_oedb_id", `CREATE UNIQUE INDEX IF NOT EXISTS idx_exercises_oedb_id ON exercises(oedb_id)`)
+ // Set when a complete English export no longer lists this oedb_id. The row is hidden
+ // from list and search but kept for every reference: workout_exercises and
+ // program_exercises FK it with no cascade. NULL means present, or never upstream.
+ ensureColumn("exercises", "upstream_missing_at", `ALTER TABLE exercises ADD COLUMN upstream_missing_at DATETIME`)
+
normalizeExerciseTaxonomy()
// The product a diary entry was, so it survives into Recent and can be matched
diff --git a/backend/db/migrations_test.go b/backend/db/migrations_test.go
index 0a89963..c1efbb7 100644
--- a/backend/db/migrations_test.go
+++ b/backend/db/migrations_test.go
@@ -980,3 +980,51 @@ func TestUpgrade_ExerciseTranslationsOnExistingData(t *testing.T) {
t.Fatal("an unknown origin was accepted")
}
}
+
+// upstream_missing_at is a nullable column on a table that workouts and routines FK
+// into, so adding it must change no row and must be safe to run on every boot.
+func TestUpgrade_ExerciseUpstreamMissingAt(t *testing.T) {
+ setupMigrationTestDB(t)
+ alterMigrations()
+ // The previous release's schema: everything but the new column.
+ if _, err := DB.Exec(`ALTER TABLE exercises DROP COLUMN upstream_missing_at`); err != nil {
+ t.Fatalf("recreate the old schema: %v", err)
+ }
+ if has, _ := hasColumn("exercises", "upstream_missing_at"); has {
+ t.Fatal("the old schema must not have the column")
+ }
+ if _, err := DB.Exec(`INSERT INTO exercises (id, name, description, oedb_id) VALUES (1, 'Upstream Row', 'desc', 'uuid-1'), (2, 'Legacy Row', 'old', NULL)`); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := DB.Exec(`INSERT INTO users (id, email, password_hash) VALUES (1, '[email protected]', 'x')`); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := DB.Exec(`INSERT INTO workouts (id, user_id, name, started_at) VALUES (1, 1, 'W', '2026-01-01 00:00:00')`); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := DB.Exec(`INSERT INTO workout_exercises (workout_id, exercise_id, order_index) VALUES (1, 1, 0)`); err != nil {
+ t.Fatal(err)
+ }
+
+ alterMigrations()
+ alterMigrations()
+
+ if has, err := hasColumn("exercises", "upstream_missing_at"); err != nil || !has {
+ t.Fatalf("hasColumn = %v, %v, want the column added", has, err)
+ }
+ if n := countRow(t, `SELECT COUNT(*) FROM exercises WHERE upstream_missing_at IS NULL`); n != 2 {
+ t.Errorf("rows with a NULL mark = %d, want 2", n)
+ }
+ if n := countRow(t, `SELECT COUNT(*) FROM exercises WHERE (id = 1 AND name = 'Upstream Row' AND description = 'desc' AND oedb_id = 'uuid-1') OR (id = 2 AND name = 'Legacy Row' AND description = 'old' AND oedb_id IS NULL)`); n != 2 {
+ t.Errorf("migrated rows = %d, want both unchanged", n)
+ }
+ if n := countRow(t, `SELECT COUNT(*) FROM workout_exercises we JOIN exercises e ON e.id = we.exercise_id`); n != 1 {
+ t.Errorf("joined references = %d, want 1", n)
+ }
+
+ setupMigrationTestDB(t)
+ alterMigrations()
+ if has, _ := hasColumn("exercises", "upstream_missing_at"); !has {
+ t.Error("a fresh database must have the column")
+ }
+}
diff --git a/backend/stores/exercise.go b/backend/stores/exercise.go
index 81bb865..fafe769 100644
--- a/backend/stores/exercise.go
+++ b/backend/stores/exercise.go
@@ -140,7 +140,7 @@ func scanExercise(row scanner, e *models.Exercise) error {
func (s *ExerciseStore) List(f ExerciseFilter) ([]models.Exercise, error) {
locale := NormalizeLocale(f.Locale)
sel, args := exerciseSelect(locale)
- q := sel + ` WHERE 1=1`
+ q := sel + ` WHERE e.upstream_missing_at IS NULL`
if f.Query != "" {
like := "%" + f.Query + "%"
q += " AND (e.name LIKE ? OR EXISTS (SELECT 1 FROM exercise_translations t WHERE t.exercise_id = e.id AND t.locale IN (?, ?) AND t.name LIKE ?))"
@@ -215,7 +215,9 @@ func (s *ExerciseStore) Count() (int, error) {
// what an English reader sees. locale is the language that was asked for, and the
// language the returned rows are shown in.
//
-// Name matching only adopts a row that no upstream id owns yet. An upstream
+// Name matching only adopts a row that no upstream id owns yet, or whose upstream
+// exercise has since been removed (wger's replaced_by: the history follows the
+// exercise that replaced it). An upstream
// exercise whose name is already owned by another id, or that arrives translated
// with no English row to attach to, is skipped and left out of the result, because oedb names are not unique and re-pointing the row would re-point
// every workout that references it.
@@ -236,7 +238,8 @@ func (s *ExerciseStore) Materialize(items []oedb.Exercise, imageBase, locale str
byOedbID, err := tx.Prepare(`
UPDATE exercises SET
name = COALESCE(?, name), muscle_group = ?, secondary_muscles = ?, category = ?,
- equipment = ?, description = COALESCE(?, description), image_url = ?, slug = ?
+ equipment = ?, description = COALESCE(?, description), image_url = ?, slug = ?,
+ upstream_missing_at = NULL
WHERE oedb_id = ?`)
if err != nil {
return nil, err
@@ -254,8 +257,9 @@ func (s *ExerciseStore) Materialize(items []oedb.Exercise, imageBase, locale str
description = COALESCE(?, description),
image_url = excluded.image_url,
oedb_id = excluded.oedb_id,
- slug = excluded.slug
- WHERE exercises.oedb_id IS NULL`)
+ slug = excluded.slug,
+ upstream_missing_at = NULL
+ WHERE exercises.oedb_id IS NULL OR exercises.upstream_missing_at IS NOT NULL`)
if err != nil {
return nil, err
}
@@ -642,6 +646,10 @@ func (s *ExerciseStore) refresh(ctx context.Context, locale string) (int, error)
}
done += len(out)
}
+ // Only after the whole export imported: a half-read export must not hide anything.
+ if err := s.markUpstreamMissing(items); err != nil {
+ return done, err
+ }
return done, nil
}
@@ -673,7 +681,7 @@ func (s *ExerciseStore) refresh(ctx context.Context, locale string) (int, error)
}
func (s *ExerciseStore) cachedOedbIDs() (map[string]bool, error) {
- rows, err := s.db.Query(`SELECT oedb_id FROM exercises WHERE oedb_id IS NOT NULL AND oedb_id != ''`)
+ rows, err := s.db.Query(`SELECT oedb_id FROM exercises WHERE oedb_id IS NOT NULL AND oedb_id != '' AND upstream_missing_at IS NULL`)
if err != nil {
return nil, err
}
@@ -689,6 +697,51 @@ func (s *ExerciseStore) cachedOedbIDs() (map[string]bool, error) {
return held, rows.Err()
}
+// markUpstreamMissing hides every held upstream row the complete English export no longer
+// lists. oedb has no deletion log, so absence from a whole export is the only signal, and
+// a truncated export looks exactly like a mass removal: when the export holds under half
+// as many exercises as the visible held rows, nothing is marked and the pass fails so it
+// is retried rather than trusted. Rows are never deleted, because workout_exercises and
+// program_exercises reference them with no cascade. One statement in one transaction, so
+// the write lock is held for a single scan of the table.
+func (s *ExerciseStore) markUpstreamMissing(items []oedb.Exercise) error {
+ ids := make([]string, len(items))
+ for i, it := range items {
+ ids[i] = it.ID
+ }
+ idsJSON, err := json.Marshal(ids)
+ if err != nil {
+ return err
+ }
+ tx, err := s.db.Begin()
+ if err != nil {
+ return err
+ }
+ defer tx.Rollback()
+
+ var held int
+ if err := tx.QueryRow(`SELECT COUNT(*) FROM exercises WHERE oedb_id IS NOT NULL AND oedb_id != '' AND upstream_missing_at IS NULL`).Scan(&held); err != nil {
+ return err
+ }
+ if held > 0 && 2*len(ids) < held {
+ log.Printf("exercises: upstream export lists %d exercises against %d held; not marking any as removed", len(ids), held)
+ return fmt.Errorf("%w: export lists %d exercises against %d held; marking none as removed", ErrCatalogUnreachable, len(ids), held)
+ }
+ res, err := tx.Exec(`UPDATE exercises SET upstream_missing_at = CURRENT_TIMESTAMP
+ WHERE oedb_id IS NOT NULL AND oedb_id != '' AND upstream_missing_at IS NULL
+ AND oedb_id NOT IN (SELECT value FROM json_each(?))`, string(idsJSON))
+ if err != nil {
+ return err
+ }
+ if err := tx.Commit(); err != nil {
+ return err
+ }
+ if n, _ := res.RowsAffected(); n > 0 {
+ log.Printf("exercises: %d exercises no longer in the upstream catalog; hidden from search, kept for history", n)
+ }
+ return nil
+}
+
// PrimeQuery materializes whatever the catalog returns for one search term.
//
// This is how a caller that needs specific exercises to exist locally — demo data
diff --git a/backend/stores/exercise_locale_sync.go b/backend/stores/exercise_locale_sync.go
index 80ed068..84a8adb 100644
--- a/backend/stores/exercise_locale_sync.go
+++ b/backend/stores/exercise_locale_sync.go
@@ -2,6 +2,7 @@ package stores
import (
"context"
+ "encoding/json"
"errors"
"fmt"
"log"
@@ -92,6 +93,8 @@ func retryDelay(failures int, every time.Duration) time.Duration {
// syncCatalog is the one catalog pass. English goes first, because it creates the rows
// every translation attaches to; the other languages follow. The int is how many
// exercises the English pass recorded. A language that fails does not stop the rest.
+// After a discovery that listed languages, translations in languages no longer offered
+// are deleted; a failed or empty discovery deletes nothing.
func (s *ExerciseStore) syncCatalog(ctx context.Context) (_ int, err error) {
if s.catalog == nil {
return 0, ErrNoCatalog
@@ -118,12 +121,22 @@ func (s *ExerciseStore) syncCatalog(ctx context.Context) (_ int, err error) {
locales = append(locales, l)
}
}
+ var errs []error
+ if len(codes) > 0 {
+ offered := make([]string, 0, len(seen))
+ for l := range seen {
+ offered = append(offered, l)
+ }
+ if err := s.purgeDroppedLocales(offered); err != nil {
+ errs = append(errs, err)
+ log.Printf("exercises: purging dropped languages failed: %.200v", err)
+ }
+ }
// replaceTranslation for es-MX deletes the es row it falls back over, so every
// regional tag must be synced before its base language.
sort.Sort(sort.Reverse(sort.StringSlice(locales)))
locales = append([]string{defaultLocale}, locales...)
english := 0
- var errs []error
for _, l := range locales {
lctx, cancel := context.WithTimeout(ctx, localeSyncTimeout)
n, err := s.refresh(lctx, l)
@@ -143,3 +156,40 @@ func (s *ExerciseStore) syncCatalog(ctx context.Context) (_ int, err error) {
}
return english, nil
}
+
+// purgeDroppedLocales deletes translations in languages oedb no longer offers. A locale
+// stays if it is offered or is the base language of an offered one, because
+// translationJoin falls back from es-MX to es. English lives in the base columns, not in
+// this table, so it is never touched.
+//
+// A discovery answer is upstream's word alone, and a wrong or truncated one would read as
+// "every language was dropped". When it would remove more than half of the languages held,
+// nothing is deleted: leftover translations are harmless, wiping them costs a full re-import.
+func (s *ExerciseStore) purgeDroppedLocales(offered []string) error {
+ keep := make([]string, 0, 2*len(offered))
+ for _, l := range offered {
+ keep = append(keep, l, BaseLanguage(l))
+ }
+ keepJSON, err := json.Marshal(keep)
+ if err != nil {
+ return err
+ }
+ var held, dropped int
+ if err := s.db.QueryRow(`SELECT COUNT(DISTINCT locale),
+ COUNT(DISTINCT CASE WHEN locale NOT IN (SELECT value FROM json_each(?1)) THEN locale END)
+ FROM exercise_translations`, string(keepJSON)).Scan(&held, &dropped); err != nil {
+ return err
+ }
+ if dropped*2 > held {
+ log.Printf("exercises: catalog now offers %d languages and would drop %d of the %d held; keeping their translations", len(offered), dropped, held)
+ return nil
+ }
+ res, err := s.db.Exec(`DELETE FROM exercise_translations WHERE locale NOT IN (SELECT value FROM json_each(?))`, string(keepJSON))
+ if err != nil {
+ return err
+ }
+ if n, _ := res.RowsAffected(); n > 0 {
+ log.Printf("exercises: removed %d translations in languages the catalog no longer offers", n)
+ }
+ return nil
+}
diff --git a/backend/stores/exercise_removal_test.go b/backend/stores/exercise_removal_test.go
new file mode 100644
index 0000000..77bb882
--- /dev/null
+++ b/backend/stores/exercise_removal_test.go
@@ -0,0 +1,380 @@
+package stores
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+ "fmt"
+ "strings"
+ "testing"
+
+ "github.com/Cawlumm/lyftr-backend/oedb"
+)
+
+func markRemoved(t *testing.T, conn *sql.DB, oedbIDs ...string) {
+ t.Helper()
+ for _, id := range oedbIDs {
+ if _, err := conn.Exec(`UPDATE exercises SET upstream_missing_at = CURRENT_TIMESTAMP WHERE oedb_id = ?`, id); err != nil {
+ t.Fatal(err)
+ }
+ }
+}
+
+func marked(t *testing.T, conn *sql.DB) int {
+ t.Helper()
+ return countRows(t, conn, `SELECT COUNT(*) FROM exercises WHERE upstream_missing_at IS NOT NULL`)
+}
+
+func isMarked(t *testing.T, conn *sql.DB, oedbID string) bool {
+ t.Helper()
+ return countRows(t, conn, `SELECT COUNT(*) FROM exercises WHERE oedb_id = ? AND upstream_missing_at IS NOT NULL`, oedbID) == 1
+}
+
+func (c *localeCatalog) setUpstream(n int) {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ c.upstream = n
+}
+
+func listedNames(t *testing.T, s *ExerciseStore, f ExerciseFilter) []string {
+ t.Helper()
+ got, err := s.List(f)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var names []string
+ for _, e := range got {
+ names = append(names, e.Name)
+ }
+ return names
+}
+
+func TestList_HidesRowsRemovedUpstream(t *testing.T) {
+ conn := testDB(t)
+ s := NewExerciseStore(conn)
+ materialize(t, s, "en", catalogExercise("uuid-a", "Alpha Press"), catalogExercise("uuid-b", "Bravo Press"), catalogExercise("uuid-c", "Charlie Press"))
+ if _, err := conn.Exec(`INSERT INTO exercises (name, muscle_group) VALUES ('Legacy Press', 'chest')`); err != nil {
+ t.Fatal(err)
+ }
+ materialize(t, s, "es", served("uuid-b", "Prensa Bravo", "es", []string{"Uno."}, "es"))
+ markRemoved(t, conn, "uuid-b")
+ id := localID(t, s, "uuid-b")
+
+ names := listedNames(t, s, ExerciseFilter{Limit: 100})
+ if len(names) != 3 {
+ t.Errorf("List = %v, want the three that remain", names)
+ }
+ for _, n := range names {
+ if n == "Bravo Press" {
+ t.Errorf("List returned the removed exercise: %v", names)
+ }
+ }
+ if got := listedNames(t, s, ExerciseFilter{Limit: 100, Query: "Bravo"}); len(got) != 0 {
+ t.Errorf("List(q=Bravo) = %v, want none", got)
+ }
+ if got := listedNames(t, s, ExerciseFilter{Limit: 100, Query: "Prensa", Locale: "es"}); len(got) != 0 {
+ t.Errorf("List(q=Prensa, es) matched through the translation: %v", got)
+ }
+ if got := mustGet(t, s, id, "en").Name; got != "Bravo Press" {
+ t.Errorf("Get = %q, want the removed exercise still resolvable by id", got)
+ }
+
+ got, err := s.Search(context.Background(), ExerciseFilter{Limit: 100})
+ if err != nil || len(got) != 3 {
+ t.Errorf("Search without a catalog = %d rows, %v, want 3", len(got), err)
+ }
+
+ cat := newLocaleCatalog(t, 3, "en")
+ cat.setFail("/api/v1/exercises", "")
+ cat.setFail("/api/v1/exercises/export", "")
+ s.UseCatalog(oedb.New(cat.srv.URL, "test"))
+ got, err = s.Search(context.Background(), ExerciseFilter{Limit: 100})
+ if err != nil || len(got) != 3 {
+ t.Errorf("Search on the offline fallback = %d rows, %v, want 3", len(got), err)
+ }
+ for _, e := range got {
+ if e.Name == "Bravo Press" {
+ t.Error("the offline fallback returned the removed exercise")
+ }
+ }
+}
+
+func TestMaterialize_RecordingARemovedExerciseUnhidesIt(t *testing.T) {
+ s := NewExerciseStore(testDB(t))
+ materialize(t, s, "en", catalogExercise("uuid-001", "Row"))
+ markRemoved(t, s.db, "uuid-001")
+ materialize(t, s, "en", catalogExercise("uuid-001", "Row"))
+ if isMarked(t, s.db, "uuid-001") {
+ t.Error("an English re-record must clear the mark")
+ }
+ markRemoved(t, s.db, "uuid-001")
+ materialize(t, s, "es", served("uuid-001", "Remo", "es", []string{"Uno."}, "es"))
+ if isMarked(t, s.db, "uuid-001") {
+ t.Error("a Spanish re-record must clear the mark")
+ }
+}
+
+func TestMaterialize_NewUpstreamIDAdoptsARemovedRowOfTheSameName(t *testing.T) {
+ s := NewExerciseStore(testDB(t))
+ materialize(t, s, "en", catalogExercise("uuid-a", "Bench Press"))
+ id := localID(t, s, "uuid-a")
+
+ materialize(t, s, "en", catalogExercise("uuid-b", "Bench Press"))
+ if got := countRows(t, s.db, `SELECT COUNT(*) FROM exercises WHERE oedb_id = 'uuid-b'`); got != 0 {
+ t.Fatalf("a live row must not be taken over, uuid-b rows = %d", got)
+ }
+
+ markRemoved(t, s.db, "uuid-a")
+ materialize(t, s, "en", catalogExercise("uuid-b", "Bench Press"))
+ if got := localID(t, s, "uuid-b"); got != id {
+ t.Errorf("uuid-b local id = %d, want the removed row %d", got, id)
+ }
+ if marked(t, s.db) != 0 || countRows(t, s.db, `SELECT COUNT(*) FROM exercises`) != 1 {
+ t.Error("want one visible row")
+ }
+}
+
+func TestSyncCatalog_HidesExercisesRemovedUpstream(t *testing.T) {
+ s, cat := syncStore(t, 10, "en")
+ if _, err := s.db.Exec(`INSERT INTO exercises (name) VALUES ('Legacy Row')`); err != nil {
+ t.Fatal(err)
+ }
+ uid := seedUser(t, s.db)
+ res, err := s.db.Exec(`INSERT INTO workouts (user_id, name) VALUES (?, 'W')`, uid)
+ if err != nil {
+ t.Fatal(err)
+ }
+ wid, _ := res.LastInsertId()
+ gone := localID(t, s, "uuid-009")
+ if _, err := s.db.Exec(`INSERT INTO workout_exercises (workout_id, exercise_id, order_index) VALUES (?, ?, 0)`, wid, gone); err != nil {
+ t.Fatal(err)
+ }
+ total := countRows(t, s.db, `SELECT COUNT(*) FROM exercises`)
+
+ cat.setUpstream(8)
+ if _, err := s.syncCatalog(t.Context()); err != nil {
+ t.Fatal(err)
+ }
+
+ if !isMarked(t, s.db, "uuid-008") || !isMarked(t, s.db, "uuid-009") || marked(t, s.db) != 2 {
+ t.Errorf("marked = %d, want exactly uuid-008 and uuid-009 (legacy and present rows untouched)", marked(t, s.db))
+ }
+ if got := countRows(t, s.db, `SELECT COUNT(*) FROM exercises`); got != total {
+ t.Errorf("exercise rows = %d, want %d: nothing is ever deleted", got, total)
+ }
+ names := listedNames(t, s, ExerciseFilter{Limit: 100})
+ if len(names) != 9 {
+ t.Errorf("List = %d rows, want 9 (8 present + legacy)", len(names))
+ }
+ if got := mustGet(t, s, gone, "en").Name; got != "Exercise 009" {
+ t.Errorf("Get = %q", got)
+ }
+ w, err := NewWorkoutStore(s.db).Get(uid, wid, "en")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(w.Exercises) != 1 || w.Exercises[0].Exercise.Name != "Exercise 009" {
+ t.Errorf("workout exercises = %+v, want the removed exercise still joined by name", w.Exercises)
+ }
+
+ cat.setUpstream(10)
+ cat.setRev(1)
+ if _, err := s.syncCatalog(t.Context()); err != nil {
+ t.Fatal(err)
+ }
+ if marked(t, s.db) != 0 {
+ t.Errorf("marked = %d after the exercises reappeared, want 0", marked(t, s.db))
+ }
+ if got := listedNames(t, s, ExerciseFilter{Limit: 100}); len(got) != 11 {
+ t.Errorf("List = %d rows, want 11", len(got))
+ }
+}
+
+func TestSyncCatalog_TruncatedExportMarksNothing(t *testing.T) {
+ s, cat := syncStore(t, 10, "en")
+ cat.setUpstream(4)
+ _, err := s.syncCatalog(t.Context())
+ if !errors.Is(err, ErrCatalogUnreachable) {
+ t.Fatalf("err = %v, want ErrCatalogUnreachable so the pass is retried", err)
+ }
+ if marked(t, s.db) != 0 {
+ t.Errorf("marked = %d, want 0", marked(t, s.db))
+ }
+
+ s2, cat2 := syncStore(t, 10, "en")
+ cat2.setUpstream(5)
+ if _, err := s2.syncCatalog(t.Context()); err != nil {
+ t.Fatalf("exactly half is allowed: %v", err)
+ }
+ if marked(t, s2.db) != 5 {
+ t.Errorf("marked = %d, want 5", marked(t, s2.db))
+ }
+}
+
+func TestSyncCatalog_EmptyExportWithHeldRowsMarksNothing(t *testing.T) {
+ s, cat := syncStore(t, 10, "en")
+ cat.setUpstream(0)
+ _, err := s.syncCatalog(t.Context())
+ if !errors.Is(err, ErrCatalogUnreachable) {
+ t.Fatalf("err = %v, want ErrCatalogUnreachable", err)
+ }
+ if marked(t, s.db) != 0 {
+ t.Errorf("marked = %d, want 0", marked(t, s.db))
+ }
+}
+
+func TestSyncCatalog_FailedEnglishImportMarksNothing(t *testing.T) {
+ s, cat := syncStore(t, 10, "en")
+ if _, err := s.db.Exec(`CREATE TRIGGER fail_import BEFORE UPDATE ON exercises WHEN NEW.oedb_id = 'uuid-003' BEGIN SELECT RAISE(ABORT, 'boom'); END`); err != nil {
+ t.Fatal(err)
+ }
+ cat.setUpstream(8)
+ if _, err := s.syncCatalog(t.Context()); err == nil {
+ t.Fatal("want the failed import reported")
+ }
+ if marked(t, s.db) != 0 {
+ t.Errorf("marked = %d, want 0: no mark unless the whole export imported", marked(t, s.db))
+ }
+}
+
+func TestSyncCatalog_RemovalMarkIsIdempotent(t *testing.T) {
+ s, cat := syncStore(t, 10, "en")
+ cat.setUpstream(8)
+ if _, err := s.syncCatalog(t.Context()); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := s.db.Exec(`UPDATE exercises SET upstream_missing_at = '2000-01-01 00:00:00' WHERE oedb_id = 'uuid-009'`); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := s.syncCatalog(t.Context()); err != nil {
+ t.Fatal(err)
+ }
+ if got := countRows(t, s.db, `SELECT COUNT(*) FROM exercises WHERE oedb_id = 'uuid-009' AND upstream_missing_at = '2000-01-01 00:00:00'`); got != 1 {
+ t.Error("a second pass must keep the first timestamp")
+ }
+ if marked(t, s.db) != 2 {
+ t.Errorf("marked = %d, want 2", marked(t, s.db))
+ }
+}
+
+func TestSyncCatalog_SkipsTranslationReadsForRemovedExercises(t *testing.T) {
+ s, cat := syncStore(t, 10, "en", "es")
+ cat.setUpstream(8)
+ if _, err := s.syncCatalog(t.Context()); err != nil {
+ t.Fatal(err)
+ }
+ for _, c := range cat.snapshot() {
+ if c.path == "/api/v1/exercises" && (strings.Contains(c.ids, "uuid-008") || strings.Contains(c.ids, "uuid-009")) {
+ t.Errorf("re-read %q for a removed exercise", c.ids)
+ }
+ }
+ if !isMarked(t, s.db, "uuid-008") || !isMarked(t, s.db, "uuid-009") {
+ t.Error("the translation pass must not un-hide a removed exercise")
+ }
+}
+
+func TestSyncCatalog_PurgesTranslationsOfDroppedLanguages(t *testing.T) {
+ s, cat := syncStore(t, 10, "en", "es", "fr")
+ if _, err := s.syncCatalog(t.Context()); err != nil {
+ t.Fatal(err)
+ }
+ if countRows(t, s.db, `SELECT COUNT(*) FROM exercise_translations WHERE locale = 'fr'`) != 10 {
+ t.Fatal("fr should be held before the language is dropped")
+ }
+ base := countRows(t, s.db, `SELECT COUNT(*) FROM exercises WHERE name LIKE 'Exercise %' AND description <> ''`)
+
+ cat.setLocales("en", "es-MX")
+ cat.setRev(1)
+ if _, err := s.syncCatalog(t.Context()); err != nil {
+ t.Fatal(err)
+ }
+ if got := countRows(t, s.db, `SELECT COUNT(*) FROM exercise_translations WHERE locale = 'fr'`); got != 0 {
+ t.Errorf("fr translations = %d, want 0", got)
+ }
+ if got := countRows(t, s.db, `SELECT COUNT(*) FROM exercise_translations WHERE locale = 'es-MX'`); got != 10 {
+ t.Errorf("es-MX translations = %d, want 10", got)
+ }
+ if got := countRows(t, s.db, `SELECT COUNT(*) FROM exercises WHERE name LIKE 'Exercise %' AND description <> ''`); got != base {
+ t.Errorf("English rows = %d, want %d untouched", got, base)
+ }
+}
+
+func TestPurgeDroppedLocales_KeepsTheBaseOfAnOfferedRegion(t *testing.T) {
+ s := NewExerciseStore(testDB(t))
+ materialize(t, s, "en", catalogExercise("uuid-1", "One"))
+ id := localID(t, s, "uuid-1")
+ for _, l := range []string{"es", "es-MX", "pt-BR", "fr"} {
+ if _, err := s.db.Exec(`INSERT INTO exercise_translations (exercise_id, locale, name) VALUES (?, ?, ?)`, id, l, l); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := s.purgeDroppedLocales([]string{"es-MX", "pt"}); err != nil {
+ t.Fatal(err)
+ }
+ var got []string
+ rows, _ := s.db.Query(`SELECT locale FROM exercise_translations ORDER BY locale`)
+ for rows.Next() {
+ var l string
+ rows.Scan(&l)
+ got = append(got, l)
+ }
+ rows.Close()
+ if fmt.Sprint(got) != "[es es-MX]" {
+ t.Errorf("locales left = %v, want [es es-MX]", got)
+ }
+}
+
+func TestSyncCatalog_KeepsTranslationsWhenDiscoveryFailsOrIsEmpty(t *testing.T) {
+ s, cat := syncStore(t, 5, "en", "es", "fr")
+ if _, err := s.syncCatalog(t.Context()); err != nil {
+ t.Fatal(err)
+ }
+ before := countRows(t, s.db, `SELECT COUNT(*) FROM exercise_translations`)
+ if before != 10 {
+ t.Fatalf("translations = %d, want 10", before)
+ }
+
+ cat.setFail("/api/v1/meta/locales", "")
+ if _, err := s.syncCatalog(t.Context()); err == nil {
+ t.Fatal("want the discovery failure reported")
+ }
+ if got := countRows(t, s.db, `SELECT COUNT(*) FROM exercise_translations`); got != before {
+ t.Errorf("translations = %d after a failed discovery, want %d", got, before)
+ }
+
+ cat.mu.Lock()
+ delete(cat.failPath, "/api/v1/meta/locales")
+ cat.mu.Unlock()
+ cat.setLocales()
+ if _, err := s.syncCatalog(t.Context()); err != nil {
+ t.Fatal(err)
+ }
+ if got := countRows(t, s.db, `SELECT COUNT(*) FROM exercise_translations`); got != before {
+ t.Errorf("translations = %d after an empty discovery, want %d", got, before)
+ }
+}
+
+// One discovery answer must not be able to wipe the translations: dropping most of the
+// held languages at once is far more likely a bad answer than a catalog that shrank.
+func TestPurgeDroppedLocales_RefusesToDropMostLanguagesAtOnce(t *testing.T) {
+ s := NewExerciseStore(testDB(t))
+ materialize(t, s, "en", catalogExercise("uuid-1", "One"))
+ id := localID(t, s, "uuid-1")
+ for _, l := range []string{"es", "fr", "de"} {
+ if _, err := s.db.Exec(`INSERT INTO exercise_translations (exercise_id, locale, name) VALUES (?, ?, ?)`, id, l, l); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := s.purgeDroppedLocales([]string{"es"}); err != nil {
+ t.Fatal(err)
+ }
+ if n := countRows(t, s.db, `SELECT COUNT(*) FROM exercise_translations`); n != 3 {
+ t.Errorf("translations = %d, want all 3 kept when 2 of 3 languages would go", n)
+ }
+ if err := s.purgeDroppedLocales([]string{"es", "fr"}); err != nil {
+ t.Fatal(err)
+ }
+ if n := countRows(t, s.db, `SELECT COUNT(*) FROM exercise_translations`); n != 2 {
+ t.Errorf("translations = %d, want 2 after dropping 1 of 3", n)
+ }
+}
diff --git a/docs/API_CHANGES.md b/docs/API_CHANGES.md
index b91170d..7f13e14 100644
--- a/docs/API_CHANGES.md
+++ b/docs/API_CHANGES.md
@@ -14,6 +14,18 @@ catches what the author knew and forgot; it cannot catch what the author never n
If your PR changes a status code, narrows an accepted value, or renames a response field,
add the entry in that PR.
+## `GET /api/v1/exercises` no longer lists exercises removed from the upstream catalog
+
+When a complete daily catalog pass no longer finds an exercise in open-exercise-db, it is left
+out of list and search results (the offline fallback included), but it is never deleted.
+`GET /exercises/:id`, its PRs and history, and every workout and routine that uses it still
+return it, and logging a workout with it still works. It comes back if upstream lists it
+again. A pass whose export is empty or holds under half the held catalog marks nothing and
+is reported as failed (`POST /admin/sync-exercises` answers 503 `EXERCISE_CATALOG_UNREACHABLE`),
+so it is retried. Translations in a language the catalog stops offering are removed after a
+successful locale discovery, unless that would drop more than half of the languages held, in which case nothing is removed; the English text is unaffected. `/admin/seed-status` `count`
+still includes hidden rows.
+
## `GET /api/v1/admin/seed-status` — gains sync status (additive)
`count` is unchanged. The response now also carries `syncing` (a catalog pass is running),