Skip to content

armgit / lyftr

Public Imported from GitHub
0

Self-hosted fitness tracker. Log workouts, track nutrition, and monitor progress. Docker deployment, mobile-friendly.

Let an operator confirm a genuine mass removal from the catalog (#216)

Carterbd9c554e6900
diff --git a/backend/controllers/exercises.go b/backend/controllers/exercises.go
index b912be3..bc83883 100644
--- a/backend/controllers/exercises.go
+++ b/backend/controllers/exercises.go
@@ -103,9 +103,14 @@ func (h *Handler) GetExerciseHistory(c *gin.Context) {
// RefreshExerciseCache runs the scheduled catalog pass now, importing every upstream
// exercise in English and refreshing translations for every held one. `refreshed` is
-// how many exercises the English pass recorded.
+// how many exercises the English pass recorded. `confirm_removals=true` accepts one pass
+// in which the catalog lists under half of the exercises held.
func (h *Handler) RefreshExerciseCache(c *gin.Context) {
- n, err := h.s.Exercise.RefreshCached(c.Request.Context())
+ refresh := h.s.Exercise.RefreshCached
+ if c.Query("confirm_removals") == "true" {
+ refresh = h.s.Exercise.RefreshCachedConfirmed
+ }
+ n, err := refresh(c.Request.Context())
if errors.Is(err, stores.ErrNoCatalog) {
utils.Fail(c, utils.CodeNoExerciseCatalog, nil)
return
@@ -114,6 +119,11 @@ func (h *Handler) RefreshExerciseCache(c *gin.Context) {
utils.Fail(c, utils.CodeExerciseSyncRunning, nil)
return
}
+ if errors.Is(err, stores.ErrCatalogShrank) {
+ log.Printf("exercises: refresh refused a shrunken catalog: %v", err)
+ utils.Fail(c, utils.CodeExerciseCatalogShrank, nil)
+ return
+ }
if errors.Is(err, stores.ErrCatalogUnreachable) {
log.Printf("exercises: refresh could not reach the catalog: %v", err)
utils.Fail(c, utils.CodeCatalogUnreachable, nil)
diff --git a/backend/controllers/food_codes_test.go b/backend/controllers/food_codes_test.go
index b3b1195..dd21a15 100644
--- a/backend/controllers/food_codes_test.go
+++ b/backend/controllers/food_codes_test.go
@@ -3,6 +3,7 @@ package controllers
import (
"context"
"errors"
+ "fmt"
"net/http"
"net/http/httptest"
"strings"
@@ -205,6 +206,36 @@ func TestRefreshExerciseCache_catalogFailureCodes(t *testing.T) {
})
}
+func TestRefreshExerciseCache_shrunkenCatalogNeedsConfirmation(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if strings.Contains(r.URL.Path, "locales") {
+ _, _ = w.Write([]byte(`{"locales":[],"total":0}`))
+ return
+ }
+ _, _ = w.Write([]byte(`{"exercises":[{"id":"uuid-1","slug":"held","muscle_group":"chest","translation":{"name":"Held","instructions":[]}}],"total":1}`))
+ }))
+ defer srv.Close()
+
+ setupTestDB(t)
+ uid := createTestUser(t)
+ for i := 1; i <= 4; i++ {
+ if _, err := db.DB.Exec("INSERT INTO exercises (name, oedb_id) VALUES (?, ?)", fmt.Sprintf("Held %d", i), fmt.Sprintf("uuid-%d", i)); err != nil {
+ t.Fatal(err)
+ }
+ }
+ th.s.Exercise.UseCatalog(oedb.New(srv.URL, "test"))
+
+ c, w := newContext(uid, http.MethodPost, "/api/v1/admin/sync-exercises", nil)
+ th.RefreshExerciseCache(c)
+ wantCode(t, w, http.StatusConflict, "EXERCISE_CATALOG_SHRANK", "The exercise catalog lists far fewer exercises than this server holds, so none were removed. If that is right, a server operator can confirm it with confirm_removals=true.")
+
+ c, w = newContext(uid, http.MethodPost, "/api/v1/admin/sync-exercises?confirm_removals=true", nil)
+ th.RefreshExerciseCache(c)
+ if w.Code != http.StatusOK {
+ t.Fatalf("confirmed: got %d %s, want 200", w.Code, w.Body.String())
+ }
+}
+
func TestRefreshExerciseCache_success(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte(`{"exercises":[{"id":"uuid-1","slug":"held","muscle_group":"chest","translation":{"name":"Held Renamed","instructions":[]}},{"id":"uuid-2","slug":"new","muscle_group":"back","translation":{"name":"New One","instructions":[]}}],"total":2}`))
diff --git a/backend/stores/exercise.go b/backend/stores/exercise.go
index 3d1f0a8..339c0d8 100644
--- a/backend/stores/exercise.go
+++ b/backend/stores/exercise.go
@@ -586,6 +586,11 @@ var ErrCatalogUnreachable = errors.New("exercise catalog unreachable")
// ErrSyncRunning is returned when a catalog pass is already in progress.
var ErrSyncRunning = errors.New("exercise sync already running")
+// ErrCatalogShrank marks an English export that lists under half of the visible held
+// exercises. It is indistinguishable from a truncated read, so the scheduled pass refuses it;
+// an operator who knows the removal is real accepts it with a confirmed sync.
+var ErrCatalogShrank = errors.New("exercise catalog shrank")
+
// CacheStatus reports how many catalog rows this instance currently holds and how the
// sync passes are going.
//
@@ -616,6 +621,13 @@ func (s *ExerciseStore) RefreshCached(ctx context.Context) (int, error) {
return s.syncCatalog(ctx)
}
+// RefreshCachedConfirmed is RefreshCached for an operator who has checked that the upstream
+// catalog really did lose most of its exercises: this one pass accepts a shrunken export
+// instead of failing with ErrCatalogShrank. An empty export is still refused.
+func (s *ExerciseStore) RefreshCachedConfirmed(ctx context.Context) (int, error) {
+ return s.syncCatalogWith(ctx, true)
+}
+
// refresh brings one language up to date. English imports every exercise in the bulk
// export, oedb.MaxPerPage rows per transaction so a pass never holds the single
// connection for long. Any other language re-reads every held row by id in oedb's
@@ -627,7 +639,7 @@ func (s *ExerciseStore) RefreshCached(ctx context.Context) (int, error) {
//
// It reads through Revalidating, because a re-read that a cache can answer re-applies
// what this instance already holds.
-func (s *ExerciseStore) refresh(ctx context.Context, locale string) (int, error) {
+func (s *ExerciseStore) refresh(ctx context.Context, locale string, acceptShrink bool) (int, error) {
if s.catalog == nil {
return 0, ErrNoCatalog
}
@@ -647,7 +659,7 @@ func (s *ExerciseStore) refresh(ctx context.Context, locale string) (int, error)
done += len(out)
}
// Only after the whole export imported: a half-read export must not hide anything.
- if err := s.markUpstreamMissing(items); err != nil {
+ if err := s.markUpstreamMissing(items, acceptShrink); err != nil {
return done, err
}
return done, nil
@@ -700,11 +712,13 @@ func (s *ExerciseStore) cachedOedbIDs() (map[string]bool, error) {
// markUpstreamMissing hides every held upstream row the complete English export no longer
// lists. oedb has no deletion log, so absence from a whole export is the only signal, and
// a truncated export looks exactly like a mass removal: when the export holds under half
-// as many exercises as the visible held rows, nothing is marked and the pass fails so it
-// is retried rather than trusted. Rows are never deleted, because workout_exercises and
+// as many exercises as the visible held rows, nothing is marked and the pass fails with
+// ErrCatalogShrank so it is retried rather than trusted, unless acceptShrink says an
+// operator confirmed the removal. An empty export is never accepted.
+// Rows are never deleted, because workout_exercises and
// program_exercises reference them with no cascade. One statement in one transaction, so
// the write lock is held for a single scan of the table.
-func (s *ExerciseStore) markUpstreamMissing(items []oedb.Exercise) error {
+func (s *ExerciseStore) markUpstreamMissing(items []oedb.Exercise, acceptShrink bool) error {
ids := make([]string, len(items))
for i, it := range items {
ids[i] = it.ID
@@ -723,9 +737,12 @@ func (s *ExerciseStore) markUpstreamMissing(items []oedb.Exercise) error {
if err := tx.QueryRow(`SELECT COUNT(*) FROM exercises WHERE oedb_id IS NOT NULL AND oedb_id != '' AND upstream_missing_at IS NULL`).Scan(&held); err != nil {
return err
}
- if held > 0 && 2*len(ids) < held {
+ if held > 0 && (len(ids) == 0 || (2*len(ids) < held && !acceptShrink)) {
log.Printf("exercises: upstream export lists %d exercises against %d held; not marking any as removed", len(ids), held)
- return fmt.Errorf("%w: export lists %d exercises against %d held; marking none as removed", ErrCatalogUnreachable, len(ids), held)
+ if len(ids) == 0 {
+ return fmt.Errorf("%w: export is empty against %d held; marking none as removed", ErrCatalogUnreachable, held)
+ }
+ return fmt.Errorf("%w: export lists %d exercises against %d held; marking none as removed", ErrCatalogShrank, len(ids), held)
}
res, err := tx.Exec(`UPDATE exercises SET upstream_missing_at = CURRENT_TIMESTAMP
WHERE oedb_id IS NOT NULL AND oedb_id != '' AND upstream_missing_at IS NULL
diff --git a/backend/stores/exercise_locale_sync.go b/backend/stores/exercise_locale_sync.go
index 84a8adb..51cdcc5 100644
--- a/backend/stores/exercise_locale_sync.go
+++ b/backend/stores/exercise_locale_sync.go
@@ -95,7 +95,12 @@ func retryDelay(failures int, every time.Duration) time.Duration {
// exercises the English pass recorded. A language that fails does not stop the rest.
// After a discovery that listed languages, translations in languages no longer offered
// are deleted; a failed or empty discovery deletes nothing.
-func (s *ExerciseStore) syncCatalog(ctx context.Context) (_ int, err error) {
+func (s *ExerciseStore) syncCatalog(ctx context.Context) (int, error) {
+ return s.syncCatalogWith(ctx, false)
+}
+
+// syncCatalogWith is syncCatalog with the shrink guard optionally lifted for this pass.
+func (s *ExerciseStore) syncCatalogWith(ctx context.Context, acceptShrink bool) (_ int, err error) {
if s.catalog == nil {
return 0, ErrNoCatalog
}
@@ -139,7 +144,7 @@ func (s *ExerciseStore) syncCatalog(ctx context.Context) (_ int, err error) {
english := 0
for _, l := range locales {
lctx, cancel := context.WithTimeout(ctx, localeSyncTimeout)
- n, err := s.refresh(lctx, l)
+ n, err := s.refresh(lctx, l, acceptShrink)
cancel()
if l == defaultLocale {
english = n
diff --git a/backend/stores/exercise_locale_sync_test.go b/backend/stores/exercise_locale_sync_test.go
index c14aa52..0405f6f 100644
--- a/backend/stores/exercise_locale_sync_test.go
+++ b/backend/stores/exercise_locale_sync_test.go
@@ -197,7 +197,7 @@ func localID(t *testing.T, s *ExerciseStore, oedbID string) int64 {
func TestRefresh_FillsEveryHeldRowInBatches(t *testing.T) {
s, cat := syncStore(t, 250)
- if _, err := s.refresh(t.Context(), "es"); err != nil {
+ if _, err := s.refresh(t.Context(), "es", false); err != nil {
t.Fatal(err)
}
if got := cat.listCount("es"); got != 3 {
diff --git a/backend/stores/exercise_removal_test.go b/backend/stores/exercise_removal_test.go
index 418eff9..b9b4c1a 100644
--- a/backend/stores/exercise_removal_test.go
+++ b/backend/stores/exercise_removal_test.go
@@ -199,8 +199,8 @@ func TestSyncCatalog_TruncatedExportMarksNothing(t *testing.T) {
s, cat := syncStore(t, 10, "en")
cat.setUpstream(4)
_, err := s.syncCatalog(t.Context())
- if !errors.Is(err, ErrCatalogUnreachable) {
- t.Fatalf("err = %v, want ErrCatalogUnreachable so the pass is retried", err)
+ if !errors.Is(err, ErrCatalogShrank) || errors.Is(err, ErrCatalogUnreachable) {
+ t.Fatalf("err = %v, want ErrCatalogShrank (retried, but not an outage)", err)
}
if marked(t, s.db) != 0 {
t.Errorf("marked = %d, want 0", marked(t, s.db))
@@ -216,6 +216,34 @@ func TestSyncCatalog_TruncatedExportMarksNothing(t *testing.T) {
}
}
+func TestRefreshCachedConfirmed_AcceptsAShrunkenExport(t *testing.T) {
+ s, cat := syncStore(t, 10, "en")
+ cat.setUpstream(4)
+ if _, err := s.RefreshCached(t.Context()); !errors.Is(err, ErrCatalogShrank) {
+ t.Fatalf("unconfirmed err = %v, want ErrCatalogShrank", err)
+ }
+ if _, err := s.RefreshCachedConfirmed(t.Context()); err != nil {
+ t.Fatalf("confirmed pass: %v", err)
+ }
+ if marked(t, s.db) != 6 {
+ t.Errorf("marked = %d, want 6", marked(t, s.db))
+ }
+ if _, err := s.syncCatalog(t.Context()); err != nil {
+ t.Errorf("the next scheduled pass must see 4 against 4 visible: %v", err)
+ }
+}
+
+func TestRefreshCachedConfirmed_StillRefusesAnEmptyExport(t *testing.T) {
+ s, cat := syncStore(t, 10, "en")
+ cat.setUpstream(0)
+ if _, err := s.RefreshCachedConfirmed(t.Context()); !errors.Is(err, ErrCatalogUnreachable) {
+ t.Fatalf("err = %v, want ErrCatalogUnreachable", err)
+ }
+ if marked(t, s.db) != 0 {
+ t.Errorf("marked = %d, want 0", marked(t, s.db))
+ }
+}
+
func TestSyncCatalog_EmptyExportWithHeldRowsMarksNothing(t *testing.T) {
s, cat := syncStore(t, 10, "en")
cat.setUpstream(0)
diff --git a/backend/utils/codes.go b/backend/utils/codes.go
index 8529135..304103d 100644
--- a/backend/utils/codes.go
+++ b/backend/utils/codes.go
@@ -116,6 +116,7 @@ var (
CodeNoExerciseCatalog = define("NO_EXERCISE_CATALOG", 400, "This server has no exercise catalog configured.")
CodeCatalogUnreachable = define("EXERCISE_CATALOG_UNREACHABLE", 503, "Couldn't reach the exercise catalog.")
CodeExerciseSyncRunning = define("EXERCISE_SYNC_RUNNING", 409, "An exercise sync is already running. Try again when it finishes.")
+ CodeExerciseCatalogShrank = define("EXERCISE_CATALOG_SHRANK", 409, "The exercise catalog lists far fewer exercises than this server holds, so none were removed. If that is right, a server operator can confirm it with confirm_removals=true.")
CodeDisplayNameControlChars = define("DISPLAY_NAME_CONTROL_CHARS", 400, "A name can't contain line breaks or other control characters.")
CodeTimezoneEmpty = define("TIMEZONE_EMPTY", 400, "Timezone can't be empty.")
CodeUnknownTimezone = define("UNKNOWN_TIMEZONE", 400, "Unknown timezone: {{timezone}}.")
diff --git a/docs/API_CHANGES.md b/docs/API_CHANGES.md
index 7f13e14..d78ba2b 100644
--- a/docs/API_CHANGES.md
+++ b/docs/API_CHANGES.md
@@ -50,6 +50,12 @@ takes longer, and answers 409 `EXERCISE_SYNC_RUNNING` if a pass (this call or th
`EXERCISE_CATALOG_UNREACHABLE` (upstream) or a database error code. `GET
/admin/seed-status` `count` is the whole catalog once a pass has completed.
+When the catalog lists fewer than half of the exercises the server holds, the pass cannot tell
+a real mass removal from a truncated read, so it hides nothing and answers 409
+`EXERCISE_CATALOG_SHRANK` (the scheduled pass logs the same and retries). Once the removal is
+confirmed, `POST /admin/sync-exercises?confirm_removals=true` accepts that one pass. The query
+parameter is additive; an empty catalog is still refused.
+
## Every error response — gains a stable `code` beside `error`
Additive: `error` is byte-identical to before and still authoritative for any caller that
@@ -57,7 +63,7 @@ ignores `code`. Every error body is now `{"error": "...", "code": "UPPER_SNAKE"}
optional `params` object when the sentence has variables. The first-party clients key their
own (localized) wording on `code` and fall back to `error`.
-Codes (72). Every error has its own code with a fixed status; there is no generic
+Codes (73). Every error has its own code with a fixed status; there is no generic
`BAD_REQUEST`, `NOT_FOUND` or `SERVICE_UNAVAILABLE`.
- Sign-in and session: `INVALID_CREDENTIALS`, `EMAIL_TAKEN`, `REGISTRATION_CLOSED`,
@@ -76,7 +82,7 @@ Codes (72). Every error has its own code with a fixed status; there is no generi
- Resources: `INVALID_ID`, `INVALID_EXERCISE_ID`, `INVALID_PROGRAM_ID`, `INVALID_WORKOUT_ID`
(`400`); `ENTRY_NOT_FOUND`, `SAVED_FOOD_NOT_FOUND`, `EXERCISE_NOT_FOUND`,
`PROGRAM_NOT_FOUND`, `WORKOUT_NOT_FOUND` (`404`); `UNKNOWN_EXERCISES`, `NO_EXERCISE_CATALOG`
- (`400`); `EXERCISE_CATALOG_UNREACHABLE` (`503`, with `Retry-After: 2`); `EXERCISE_SYNC_RUNNING` (`409`).
+ (`400`); `EXERCISE_CATALOG_UNREACHABLE` (`503`, with `Retry-After: 2`); `EXERCISE_SYNC_RUNNING`, `EXERCISE_CATALOG_SHRANK` (`409`).
- Dates: `INVALID_DATE`, `INVALID_LOGGED_ON`, `INVALID_FROM_DATE`, `INVALID_TO_DATE` (`400`).
- Food: `FOOD_NAME_TOO_LONG`, `FOOD_BRAND_TOO_LONG`, `SERVING_SIZE_TOO_LONG`,
`BARCODE_TOO_LONG`, `IMAGE_URL_TOO_LONG`, `SEARCH_TERM_REQUIRED`, `INVALID_BARCODE` (`400`);
diff --git a/packages/shared/src/client.test.ts b/packages/shared/src/client.test.ts
index ab460fd..06646ab 100644
--- a/packages/shared/src/client.test.ts
+++ b/packages/shared/src/client.test.ts
@@ -690,6 +690,7 @@ describe('apiErrorMessage — server codes', () => {
['FOOD_DATABASE_TIMEOUT', 503, undefined, undefined],
['EXERCISE_CATALOG_UNREACHABLE', 503, undefined, undefined],
['EXERCISE_SYNC_RUNNING', 409, undefined, undefined],
+ ['EXERCISE_CATALOG_SHRANK', 409, undefined, undefined],
['PRODUCT_NOT_FOUND', 404, undefined, undefined],
['SEARCH_TERM_REQUIRED', 400, undefined, undefined],
['INVALID_WORKOUT_ID', 400, undefined, undefined],
diff --git a/packages/shared/src/i18n/locales/en.json b/packages/shared/src/i18n/locales/en.json
index 21f14c2..162d4a7 100644
--- a/packages/shared/src/i18n/locales/en.json
+++ b/packages/shared/src/i18n/locales/en.json
@@ -360,6 +360,7 @@
"BARCODE_LOOKUP_UNAVAILABLE": "Barcode lookup is temporarily unavailable.",
"NO_EXERCISE_CATALOG": "This server has no exercise catalog configured.",
"EXERCISE_CATALOG_UNREACHABLE": "Couldn't reach the exercise catalog.",
+ "EXERCISE_CATALOG_SHRANK": "The exercise catalog lists far fewer exercises than this server holds, so none were removed. If that is right, a server operator can confirm it with confirm_removals=true.",
"EXERCISE_SYNC_RUNNING": "An exercise sync is already running. Try again when it finishes.",
"DISPLAY_NAME_CONTROL_CHARS": "A name can't contain line breaks or other control characters.",
"TIMEZONE_EMPTY": "Timezone can't be empty.",
diff --git a/packages/shared/src/i18n/locales/es.json b/packages/shared/src/i18n/locales/es.json
index afb8876..7162378 100644
--- a/packages/shared/src/i18n/locales/es.json
+++ b/packages/shared/src/i18n/locales/es.json
@@ -366,6 +366,7 @@
"BARCODE_LOOKUP_UNAVAILABLE": "La búsqueda por código de barras no está disponible por ahora.",
"NO_EXERCISE_CATALOG": "Este servidor no tiene un catálogo de ejercicios configurado.",
"EXERCISE_CATALOG_UNREACHABLE": "No se pudo conectar con el catálogo de ejercicios.",
+ "EXERCISE_CATALOG_SHRANK": "El catálogo de ejercicios lista muchos menos ejercicios de los que tiene este servidor, así que no se quitó ninguno. Si es correcto, quien administre el servidor puede confirmarlo con confirm_removals=true.",
"EXERCISE_SYNC_RUNNING": "Ya hay una sincronización de ejercicios en curso. Inténtalo de nuevo cuando termine.",
"DISPLAY_NAME_CONTROL_CHARS": "Un nombre no puede contener saltos de línea ni otros caracteres de control.",
"TIMEZONE_EMPTY": "La zona horaria no puede estar vacía.",