Self-hosted fitness tracker. Log workouts, track nutrition, and monitor progress. Docker deployment, mobile-friendly.
Let an operator confirm a genuine mass removal from the catalog (#216)
Carterbd9c554e6900
diff --git a/backend/controllers/exercises.go b/backend/controllers/exercises.goindex b912be3..bc83883 100644--- a/backend/controllers/exercises.go+++ b/backend/controllers/exercises.go@@ -103,9 +103,14 @@ func (h *Handler) GetExerciseHistory(c *gin.Context) {// RefreshExerciseCache runs the scheduled catalog pass now, importing every upstream// exercise in English and refreshing translations for every held one. `refreshed` is-// how many exercises the English pass recorded.+// how many exercises the English pass recorded. `confirm_removals=true` accepts one pass+// in which the catalog lists under half of the exercises held.func (h *Handler) RefreshExerciseCache(c *gin.Context) {- n, err := h.s.Exercise.RefreshCached(c.Request.Context())+ refresh := h.s.Exercise.RefreshCached+ if c.Query("confirm_removals") == "true" {+ refresh = h.s.Exercise.RefreshCachedConfirmed+ }+ n, err := refresh(c.Request.Context())if errors.Is(err, stores.ErrNoCatalog) {utils.Fail(c, utils.CodeNoExerciseCatalog, nil)return@@ -114,6 +119,11 @@ func (h *Handler) RefreshExerciseCache(c *gin.Context) {utils.Fail(c, utils.CodeExerciseSyncRunning, nil)return}+ if errors.Is(err, stores.ErrCatalogShrank) {+ log.Printf("exercises: refresh refused a shrunken catalog: %v", err)+ utils.Fail(c, utils.CodeExerciseCatalogShrank, nil)+ return+ }if errors.Is(err, stores.ErrCatalogUnreachable) {log.Printf("exercises: refresh could not reach the catalog: %v", err)utils.Fail(c, utils.CodeCatalogUnreachable, nil)diff --git a/backend/controllers/food_codes_test.go b/backend/controllers/food_codes_test.goindex b3b1195..dd21a15 100644--- a/backend/controllers/food_codes_test.go+++ b/backend/controllers/food_codes_test.go@@ -3,6 +3,7 @@ package controllersimport ("context""errors"+ "fmt""net/http""net/http/httptest""strings"@@ -205,6 +206,36 @@ func TestRefreshExerciseCache_catalogFailureCodes(t *testing.T) {})}+func TestRefreshExerciseCache_shrunkenCatalogNeedsConfirmation(t *testing.T) {+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if strings.Contains(r.URL.Path, "locales") {+ _, _ = w.Write([]byte(`{"locales":[],"total":0}`))+ return+ }+ _, _ = w.Write([]byte(`{"exercises":[{"id":"uuid-1","slug":"held","muscle_group":"chest","translation":{"name":"Held","instructions":[]}}],"total":1}`))+ }))+ defer srv.Close()++ setupTestDB(t)+ uid := createTestUser(t)+ for i := 1; i <= 4; i++ {+ if _, err := db.DB.Exec("INSERT INTO exercises (name, oedb_id) VALUES (?, ?)", fmt.Sprintf("Held %d", i), fmt.Sprintf("uuid-%d", i)); err != nil {+ t.Fatal(err)+ }+ }+ th.s.Exercise.UseCatalog(oedb.New(srv.URL, "test"))++ c, w := newContext(uid, http.MethodPost, "/api/v1/admin/sync-exercises", nil)+ th.RefreshExerciseCache(c)+ wantCode(t, w, http.StatusConflict, "EXERCISE_CATALOG_SHRANK", "The exercise catalog lists far fewer exercises than this server holds, so none were removed. If that is right, a server operator can confirm it with confirm_removals=true.")++ c, w = newContext(uid, http.MethodPost, "/api/v1/admin/sync-exercises?confirm_removals=true", nil)+ th.RefreshExerciseCache(c)+ if w.Code != http.StatusOK {+ t.Fatalf("confirmed: got %d %s, want 200", w.Code, w.Body.String())+ }+}+func TestRefreshExerciseCache_success(t *testing.T) {srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {_, _ = w.Write([]byte(`{"exercises":[{"id":"uuid-1","slug":"held","muscle_group":"chest","translation":{"name":"Held Renamed","instructions":[]}},{"id":"uuid-2","slug":"new","muscle_group":"back","translation":{"name":"New One","instructions":[]}}],"total":2}`))diff --git a/backend/stores/exercise.go b/backend/stores/exercise.goindex 3d1f0a8..339c0d8 100644--- a/backend/stores/exercise.go+++ b/backend/stores/exercise.go@@ -586,6 +586,11 @@ var ErrCatalogUnreachable = errors.New("exercise catalog unreachable")// ErrSyncRunning is returned when a catalog pass is already in progress.var ErrSyncRunning = errors.New("exercise sync already running")+// ErrCatalogShrank marks an English export that lists under half of the visible held+// exercises. It is indistinguishable from a truncated read, so the scheduled pass refuses it;+// an operator who knows the removal is real accepts it with a confirmed sync.+var ErrCatalogShrank = errors.New("exercise catalog shrank")+// CacheStatus reports how many catalog rows this instance currently holds and how the// sync passes are going.//@@ -616,6 +621,13 @@ func (s *ExerciseStore) RefreshCached(ctx context.Context) (int, error) {return s.syncCatalog(ctx)}+// RefreshCachedConfirmed is RefreshCached for an operator who has checked that the upstream+// catalog really did lose most of its exercises: this one pass accepts a shrunken export+// instead of failing with ErrCatalogShrank. An empty export is still refused.+func (s *ExerciseStore) RefreshCachedConfirmed(ctx context.Context) (int, error) {+ return s.syncCatalogWith(ctx, true)+}+// refresh brings one language up to date. English imports every exercise in the bulk// export, oedb.MaxPerPage rows per transaction so a pass never holds the single// connection for long. Any other language re-reads every held row by id in oedb's@@ -627,7 +639,7 @@ func (s *ExerciseStore) RefreshCached(ctx context.Context) (int, error) {//// It reads through Revalidating, because a re-read that a cache can answer re-applies// what this instance already holds.-func (s *ExerciseStore) refresh(ctx context.Context, locale string) (int, error) {+func (s *ExerciseStore) refresh(ctx context.Context, locale string, acceptShrink bool) (int, error) {if s.catalog == nil {return 0, ErrNoCatalog}@@ -647,7 +659,7 @@ func (s *ExerciseStore) refresh(ctx context.Context, locale string) (int, error)done += len(out)}// Only after the whole export imported: a half-read export must not hide anything.- if err := s.markUpstreamMissing(items); err != nil {+ if err := s.markUpstreamMissing(items, acceptShrink); err != nil {return done, err}return done, nil@@ -700,11 +712,13 @@ func (s *ExerciseStore) cachedOedbIDs() (map[string]bool, error) {// markUpstreamMissing hides every held upstream row the complete English export no longer// lists. oedb has no deletion log, so absence from a whole export is the only signal, and// a truncated export looks exactly like a mass removal: when the export holds under half-// as many exercises as the visible held rows, nothing is marked and the pass fails so it-// is retried rather than trusted. Rows are never deleted, because workout_exercises and+// as many exercises as the visible held rows, nothing is marked and the pass fails with+// ErrCatalogShrank so it is retried rather than trusted, unless acceptShrink says an+// operator confirmed the removal. An empty export is never accepted.+// Rows are never deleted, because workout_exercises and// program_exercises reference them with no cascade. One statement in one transaction, so// the write lock is held for a single scan of the table.-func (s *ExerciseStore) markUpstreamMissing(items []oedb.Exercise) error {+func (s *ExerciseStore) markUpstreamMissing(items []oedb.Exercise, acceptShrink bool) error {ids := make([]string, len(items))for i, it := range items {ids[i] = it.ID@@ -723,9 +737,12 @@ func (s *ExerciseStore) markUpstreamMissing(items []oedb.Exercise) error {if err := tx.QueryRow(`SELECT COUNT(*) FROM exercises WHERE oedb_id IS NOT NULL AND oedb_id != '' AND upstream_missing_at IS NULL`).Scan(&held); err != nil {return err}- if held > 0 && 2*len(ids) < held {+ if held > 0 && (len(ids) == 0 || (2*len(ids) < held && !acceptShrink)) {log.Printf("exercises: upstream export lists %d exercises against %d held; not marking any as removed", len(ids), held)- return fmt.Errorf("%w: export lists %d exercises against %d held; marking none as removed", ErrCatalogUnreachable, len(ids), held)+ if len(ids) == 0 {+ return fmt.Errorf("%w: export is empty against %d held; marking none as removed", ErrCatalogUnreachable, held)+ }+ return fmt.Errorf("%w: export lists %d exercises against %d held; marking none as removed", ErrCatalogShrank, len(ids), held)}res, err := tx.Exec(`UPDATE exercises SET upstream_missing_at = CURRENT_TIMESTAMPWHERE oedb_id IS NOT NULL AND oedb_id != '' AND upstream_missing_at IS NULLdiff --git a/backend/stores/exercise_locale_sync.go b/backend/stores/exercise_locale_sync.goindex 84a8adb..51cdcc5 100644--- a/backend/stores/exercise_locale_sync.go+++ b/backend/stores/exercise_locale_sync.go@@ -95,7 +95,12 @@ func retryDelay(failures int, every time.Duration) time.Duration {// exercises the English pass recorded. A language that fails does not stop the rest.// After a discovery that listed languages, translations in languages no longer offered// are deleted; a failed or empty discovery deletes nothing.-func (s *ExerciseStore) syncCatalog(ctx context.Context) (_ int, err error) {+func (s *ExerciseStore) syncCatalog(ctx context.Context) (int, error) {+ return s.syncCatalogWith(ctx, false)+}++// syncCatalogWith is syncCatalog with the shrink guard optionally lifted for this pass.+func (s *ExerciseStore) syncCatalogWith(ctx context.Context, acceptShrink bool) (_ int, err error) {if s.catalog == nil {return 0, ErrNoCatalog}@@ -139,7 +144,7 @@ func (s *ExerciseStore) syncCatalog(ctx context.Context) (_ int, err error) {english := 0for _, l := range locales {lctx, cancel := context.WithTimeout(ctx, localeSyncTimeout)- n, err := s.refresh(lctx, l)+ n, err := s.refresh(lctx, l, acceptShrink)cancel()if l == defaultLocale {english = ndiff --git a/backend/stores/exercise_locale_sync_test.go b/backend/stores/exercise_locale_sync_test.goindex c14aa52..0405f6f 100644--- a/backend/stores/exercise_locale_sync_test.go+++ b/backend/stores/exercise_locale_sync_test.go@@ -197,7 +197,7 @@ func localID(t *testing.T, s *ExerciseStore, oedbID string) int64 {func TestRefresh_FillsEveryHeldRowInBatches(t *testing.T) {s, cat := syncStore(t, 250)- if _, err := s.refresh(t.Context(), "es"); err != nil {+ if _, err := s.refresh(t.Context(), "es", false); err != nil {t.Fatal(err)}if got := cat.listCount("es"); got != 3 {diff --git a/backend/stores/exercise_removal_test.go b/backend/stores/exercise_removal_test.goindex 418eff9..b9b4c1a 100644--- a/backend/stores/exercise_removal_test.go+++ b/backend/stores/exercise_removal_test.go@@ -199,8 +199,8 @@ func TestSyncCatalog_TruncatedExportMarksNothing(t *testing.T) {s, cat := syncStore(t, 10, "en")cat.setUpstream(4)_, err := s.syncCatalog(t.Context())- if !errors.Is(err, ErrCatalogUnreachable) {- t.Fatalf("err = %v, want ErrCatalogUnreachable so the pass is retried", err)+ if !errors.Is(err, ErrCatalogShrank) || errors.Is(err, ErrCatalogUnreachable) {+ t.Fatalf("err = %v, want ErrCatalogShrank (retried, but not an outage)", err)}if marked(t, s.db) != 0 {t.Errorf("marked = %d, want 0", marked(t, s.db))@@ -216,6 +216,34 @@ func TestSyncCatalog_TruncatedExportMarksNothing(t *testing.T) {}}+func TestRefreshCachedConfirmed_AcceptsAShrunkenExport(t *testing.T) {+ s, cat := syncStore(t, 10, "en")+ cat.setUpstream(4)+ if _, err := s.RefreshCached(t.Context()); !errors.Is(err, ErrCatalogShrank) {+ t.Fatalf("unconfirmed err = %v, want ErrCatalogShrank", err)+ }+ if _, err := s.RefreshCachedConfirmed(t.Context()); err != nil {+ t.Fatalf("confirmed pass: %v", err)+ }+ if marked(t, s.db) != 6 {+ t.Errorf("marked = %d, want 6", marked(t, s.db))+ }+ if _, err := s.syncCatalog(t.Context()); err != nil {+ t.Errorf("the next scheduled pass must see 4 against 4 visible: %v", err)+ }+}++func TestRefreshCachedConfirmed_StillRefusesAnEmptyExport(t *testing.T) {+ s, cat := syncStore(t, 10, "en")+ cat.setUpstream(0)+ if _, err := s.RefreshCachedConfirmed(t.Context()); !errors.Is(err, ErrCatalogUnreachable) {+ t.Fatalf("err = %v, want ErrCatalogUnreachable", err)+ }+ if marked(t, s.db) != 0 {+ t.Errorf("marked = %d, want 0", marked(t, s.db))+ }+}+func TestSyncCatalog_EmptyExportWithHeldRowsMarksNothing(t *testing.T) {s, cat := syncStore(t, 10, "en")cat.setUpstream(0)diff --git a/backend/utils/codes.go b/backend/utils/codes.goindex 8529135..304103d 100644--- a/backend/utils/codes.go+++ b/backend/utils/codes.go@@ -116,6 +116,7 @@ var (CodeNoExerciseCatalog = define("NO_EXERCISE_CATALOG", 400, "This server has no exercise catalog configured.")CodeCatalogUnreachable = define("EXERCISE_CATALOG_UNREACHABLE", 503, "Couldn't reach the exercise catalog.")CodeExerciseSyncRunning = define("EXERCISE_SYNC_RUNNING", 409, "An exercise sync is already running. Try again when it finishes.")+ CodeExerciseCatalogShrank = define("EXERCISE_CATALOG_SHRANK", 409, "The exercise catalog lists far fewer exercises than this server holds, so none were removed. If that is right, a server operator can confirm it with confirm_removals=true.")CodeDisplayNameControlChars = define("DISPLAY_NAME_CONTROL_CHARS", 400, "A name can't contain line breaks or other control characters.")CodeTimezoneEmpty = define("TIMEZONE_EMPTY", 400, "Timezone can't be empty.")CodeUnknownTimezone = define("UNKNOWN_TIMEZONE", 400, "Unknown timezone: {{timezone}}.")diff --git a/docs/API_CHANGES.md b/docs/API_CHANGES.mdindex 7f13e14..d78ba2b 100644--- a/docs/API_CHANGES.md+++ b/docs/API_CHANGES.md@@ -50,6 +50,12 @@ takes longer, and answers 409 `EXERCISE_SYNC_RUNNING` if a pass (this call or th`EXERCISE_CATALOG_UNREACHABLE` (upstream) or a database error code. `GET/admin/seed-status` `count` is the whole catalog once a pass has completed.+When the catalog lists fewer than half of the exercises the server holds, the pass cannot tell+a real mass removal from a truncated read, so it hides nothing and answers 409+`EXERCISE_CATALOG_SHRANK` (the scheduled pass logs the same and retries). Once the removal is+confirmed, `POST /admin/sync-exercises?confirm_removals=true` accepts that one pass. The query+parameter is additive; an empty catalog is still refused.+## Every error response — gains a stable `code` beside `error`Additive: `error` is byte-identical to before and still authoritative for any caller that@@ -57,7 +63,7 @@ ignores `code`. Every error body is now `{"error": "...", "code": "UPPER_SNAKE"}optional `params` object when the sentence has variables. The first-party clients key theirown (localized) wording on `code` and fall back to `error`.-Codes (72). Every error has its own code with a fixed status; there is no generic+Codes (73). Every error has its own code with a fixed status; there is no generic`BAD_REQUEST`, `NOT_FOUND` or `SERVICE_UNAVAILABLE`.- Sign-in and session: `INVALID_CREDENTIALS`, `EMAIL_TAKEN`, `REGISTRATION_CLOSED`,@@ -76,7 +82,7 @@ Codes (72). Every error has its own code with a fixed status; there is no generi- Resources: `INVALID_ID`, `INVALID_EXERCISE_ID`, `INVALID_PROGRAM_ID`, `INVALID_WORKOUT_ID`(`400`); `ENTRY_NOT_FOUND`, `SAVED_FOOD_NOT_FOUND`, `EXERCISE_NOT_FOUND`,`PROGRAM_NOT_FOUND`, `WORKOUT_NOT_FOUND` (`404`); `UNKNOWN_EXERCISES`, `NO_EXERCISE_CATALOG`- (`400`); `EXERCISE_CATALOG_UNREACHABLE` (`503`, with `Retry-After: 2`); `EXERCISE_SYNC_RUNNING` (`409`).+ (`400`); `EXERCISE_CATALOG_UNREACHABLE` (`503`, with `Retry-After: 2`); `EXERCISE_SYNC_RUNNING`, `EXERCISE_CATALOG_SHRANK` (`409`).- Dates: `INVALID_DATE`, `INVALID_LOGGED_ON`, `INVALID_FROM_DATE`, `INVALID_TO_DATE` (`400`).- Food: `FOOD_NAME_TOO_LONG`, `FOOD_BRAND_TOO_LONG`, `SERVING_SIZE_TOO_LONG`,`BARCODE_TOO_LONG`, `IMAGE_URL_TOO_LONG`, `SEARCH_TERM_REQUIRED`, `INVALID_BARCODE` (`400`);diff --git a/packages/shared/src/client.test.ts b/packages/shared/src/client.test.tsindex ab460fd..06646ab 100644--- a/packages/shared/src/client.test.ts+++ b/packages/shared/src/client.test.ts@@ -690,6 +690,7 @@ describe('apiErrorMessage — server codes', () => {['FOOD_DATABASE_TIMEOUT', 503, undefined, undefined],['EXERCISE_CATALOG_UNREACHABLE', 503, undefined, undefined],['EXERCISE_SYNC_RUNNING', 409, undefined, undefined],+ ['EXERCISE_CATALOG_SHRANK', 409, undefined, undefined],['PRODUCT_NOT_FOUND', 404, undefined, undefined],['SEARCH_TERM_REQUIRED', 400, undefined, undefined],['INVALID_WORKOUT_ID', 400, undefined, undefined],diff --git a/packages/shared/src/i18n/locales/en.json b/packages/shared/src/i18n/locales/en.jsonindex 21f14c2..162d4a7 100644--- a/packages/shared/src/i18n/locales/en.json+++ b/packages/shared/src/i18n/locales/en.json@@ -360,6 +360,7 @@"BARCODE_LOOKUP_UNAVAILABLE": "Barcode lookup is temporarily unavailable.","NO_EXERCISE_CATALOG": "This server has no exercise catalog configured.","EXERCISE_CATALOG_UNREACHABLE": "Couldn't reach the exercise catalog.",+ "EXERCISE_CATALOG_SHRANK": "The exercise catalog lists far fewer exercises than this server holds, so none were removed. If that is right, a server operator can confirm it with confirm_removals=true.","EXERCISE_SYNC_RUNNING": "An exercise sync is already running. Try again when it finishes.","DISPLAY_NAME_CONTROL_CHARS": "A name can't contain line breaks or other control characters.","TIMEZONE_EMPTY": "Timezone can't be empty.",diff --git a/packages/shared/src/i18n/locales/es.json b/packages/shared/src/i18n/locales/es.jsonindex afb8876..7162378 100644--- a/packages/shared/src/i18n/locales/es.json+++ b/packages/shared/src/i18n/locales/es.json@@ -366,6 +366,7 @@"BARCODE_LOOKUP_UNAVAILABLE": "La búsqueda por código de barras no está disponible por ahora.","NO_EXERCISE_CATALOG": "Este servidor no tiene un catálogo de ejercicios configurado.","EXERCISE_CATALOG_UNREACHABLE": "No se pudo conectar con el catálogo de ejercicios.",+ "EXERCISE_CATALOG_SHRANK": "El catálogo de ejercicios lista muchos menos ejercicios de los que tiene este servidor, así que no se quitó ninguno. Si es correcto, quien administre el servidor puede confirmarlo con confirm_removals=true.","EXERCISE_SYNC_RUNNING": "Ya hay una sincronización de ejercicios en curso. Inténtalo de nuevo cuando termine.","DISPLAY_NAME_CONTROL_CHARS": "Un nombre no puede contener saltos de línea ni otros caracteres de control.","TIMEZONE_EMPTY": "La zona horaria no puede estar vacía.",