Skip to content

PostgreSQL backup tool with Point-In-Time-Recovery and restore verification

FIX (storages): Grow the deletion backoff from attempts, not from requests

Rostislav Dugincb86a25b516f
diff --git a/backend/AGENTS.md b/backend/AGENTS.md
index d2819784..db04840d 100644
--- a/backend/AGENTS.md
+++ b/backend/AGENTS.md
@@ -261,7 +261,9 @@ Two rules follow from that:
and history carry an attempt UUID of their own.
- **A provider's `DeleteFile` owes every representation it derives from the name**:
chunk objects and manifests, incomplete multipart uploads, staged blocks,
- temporary and partial files. Only a confirmed absence counts as success.
+ temporary and partial files. Only a confirmed absence counts as success. It puts
+ its own deadline on the call and must stay under `Timings.AttemptLease`, or the
+ worker reclaims the row while the attempt is still running.
A test that asserts a file is gone drives the worker first, through
`storages.DrainStorageFileDeletions`. Nothing observes cleanup by sleeping.
diff --git a/backend/internal/features/storages/files/dto.go b/backend/internal/features/storages/files/dto.go
index a5ab4428..e159698a 100644
--- a/backend/internal/features/storages/files/dto.go
+++ b/backend/internal/features/storages/files/dto.go
@@ -22,8 +22,8 @@ type Dependencies struct {
}
type ClaimRequest struct {
- Limit int
- AttemptTimeout time.Duration
+ Limit int
+ AttemptLease time.Duration
// ExcludedIDs are the writes running in this process, which have an
// obligation but no owner willing to let it go yet.
ExcludedIDs []uuid.UUID
@@ -42,8 +42,8 @@ type StoredFileReference struct {
}
// Permission to publish one uploaded file. Generation is the pending row's
-// attempt count when the upload finished: anything that takes the file away from
-// its writer increments it, so a receipt that no longer matches cannot confirm.
+// generation when the upload finished: anything that takes the file away from its
+// writer raises it, so a receipt that no longer matches cannot confirm.
type WriteReceipt struct {
PendingDeletionID uuid.UUID
Reference StoredFileReference
diff --git a/backend/internal/features/storages/files/model.go b/backend/internal/features/storages/files/model.go
index 5aaba19c..5ed69cd4 100644
--- a/backend/internal/features/storages/files/model.go
+++ b/backend/internal/features/storages/files/model.go
@@ -13,9 +13,12 @@ type PendingDeletion struct {
StorageID uuid.UUID `gorm:"column:storage_id;not null;type:uuid"`
FileName string `gorm:"column:file_name;not null;type:text"`
NotBefore time.Time `gorm:"column:not_before;not null"`
- // Retry counter and receipt generation at once: every event that takes the
- // file away from its writer increments it, which invalidates the receipt the
- // writer is holding.
+ // Generation is what a write receipt matches. Every event that takes the file
+ // away from its writer raises it, which invalidates the receipt the writer is
+ // holding.
+ Generation int `gorm:"column:generation;not null"`
+ // AttemptCount counts deletion attempts and nothing else, so the backoff grows
+ // with failures rather than with how often the file was requested.
AttemptCount int `gorm:"column:attempt_count;not null"`
LastError *string `gorm:"column:last_error;type:text"`
CreatedAt time.Time `gorm:"column:created_at;not null"`
diff --git a/backend/internal/features/storages/files/repository.go b/backend/internal/features/storages/files/repository.go
index 7038a8ae..46043fa9 100644
--- a/backend/internal/features/storages/files/repository.go
+++ b/backend/internal/features/storages/files/repository.go
@@ -62,9 +62,9 @@ func (r *PendingDeletionRepository) InsertOrTake(tx *gorm.DB, references []Store
INSERT INTO storage_pending_deletions (storage_id, file_name, not_before)
VALUES %s
ON CONFLICT (storage_id, file_name) DO UPDATE
- SET not_before = now(),
- attempt_count = storage_pending_deletions.attempt_count + 1,
- updated_at = now()`, strings.Join(placeholders, ", "))
+ SET not_before = now(),
+ generation = storage_pending_deletions.generation + 1,
+ updated_at = now()`, strings.Join(placeholders, ", "))
if err := tx.Exec(statement, args...).Error; err != nil {
return fmt.Errorf("record pending deletions: %w", err)
@@ -82,7 +82,7 @@ func (r *PendingDeletionRepository) CompleteIfGeneration(
generation int,
) (bool, error) {
result := tx.
- Where("id = ? AND attempt_count = ?", id, generation).
+ Where("id = ? AND generation = ?", id, generation).
Delete(&PendingDeletion{})
if result.Error != nil {
return false, fmt.Errorf("complete pending deletion: %w", result.Error)
@@ -98,7 +98,7 @@ func (r *PendingDeletionRepository) SetNotBeforeIfGeneration(
dueIn time.Duration,
) (bool, error) {
result := tx.Model(&PendingDeletion{}).
- Where("id = ? AND attempt_count = ?", id, generation).
+ Where("id = ? AND generation = ?", id, generation).
Updates(map[string]any{
"not_before": databaseTimeIn(dueIn),
"updated_at": gorm.Expr("now()"),
@@ -118,7 +118,7 @@ func (r *PendingDeletionRepository) RescheduleIfGeneration(
lastError string,
) (bool, error) {
result := tx.Model(&PendingDeletion{}).
- Where("id = ? AND attempt_count = ?", id, generation).
+ Where("id = ? AND generation = ?", id, generation).
Updates(map[string]any{
"not_before": databaseTimeIn(dueIn),
"last_error": lastError,
@@ -131,16 +131,16 @@ func (r *PendingDeletionRepository) RescheduleIfGeneration(
return result.RowsAffected == 1, nil
}
-// Incrementing the generation invalidates any receipt still outstanding for these
-// files, and pushing not_before past the attempt keeps a stalled provider call from
-// holding a row forever. SKIP LOCKED keeps an open deletion request from stalling
-// the whole batch.
+// Raising the generation invalidates any receipt still outstanding for these files,
+// and pushing not_before by the lease keeps a stalled provider call from holding a
+// row forever. SKIP LOCKED keeps an open deletion request from stalling the whole
+// batch.
func (r *PendingDeletionRepository) ClaimDue(
tx *gorm.DB,
request ClaimRequest,
) ([]PendingDeletion, error) {
exclusion := ""
- args := []any{request.AttemptTimeout.Seconds()}
+ args := []any{request.AttemptLease.Seconds()}
if len(request.ExcludedIDs) > 0 {
exclusion = "AND id NOT IN (?)"
@@ -152,7 +152,8 @@ func (r *PendingDeletionRepository) ClaimDue(
statement := fmt.Sprintf(`
UPDATE storage_pending_deletions
- SET attempt_count = attempt_count + 1,
+ SET generation = generation + 1,
+ attempt_count = attempt_count + 1,
not_before = now() + (? * interval '1 second'),
updated_at = now()
WHERE id IN (
diff --git a/backend/internal/features/storages/files/repository_test.go b/backend/internal/features/storages/files/repository_test.go
index 03bd45c8..9801e295 100644
--- a/backend/internal/features/storages/files/repository_test.go
+++ b/backend/internal/features/storages/files/repository_test.go
@@ -45,6 +45,7 @@ func Test_InsertIfAbsent_WhenNameIsFree_RegistersObligation(t *testing.T) {
pending, err := repository.InsertIfAbsent(db.GetDb(), uuid.New(), reference, time.Hour)
require.NoError(t, err)
+ assert.Equal(t, 0, pending.Generation)
assert.Equal(t, 0, pending.AttemptCount)
assert.Equal(t, reference, pending.GetReference())
}
@@ -90,7 +91,9 @@ func Test_InsertOrTake_WhenRequestedTwice_KeepsOneRowAndRaisesGeneration(t *test
require.NoError(t, err)
require.NotNil(t, found)
- assert.Equal(t, 1, found.AttemptCount)
+ assert.Equal(t, 1, found.Generation)
+ assert.Equal(t, 0, found.AttemptCount,
+ "asking for a file again is not a deletion attempt and must not advance the backoff")
}
func Test_InsertOrTake_WhenBatchRepeatsAReference_DoesNotFail(t *testing.T) {
@@ -104,7 +107,7 @@ func Test_InsertOrTake_WhenBatchRepeatsAReference_DoesNotFail(t *testing.T) {
found, findErr := repository.FindByReference(db.GetDb(), reference)
require.NoError(t, findErr)
require.NotNil(t, found)
- assert.Equal(t, 0, found.AttemptCount)
+ assert.Equal(t, 0, found.Generation)
}
func Test_InsertOrTake_WhenBatchIsEmpty_DoesNothing(t *testing.T) {
@@ -122,7 +125,7 @@ func Test_InsertOrTake_WhenWriterHoldsTheRow_TakesItByRaisingGeneration(t *testi
require.NoError(t, repository.InsertOrTake(db.GetDb(), []StoredFileReference{reference}))
- completed, err := repository.CompleteIfGeneration(db.GetDb(), pending.ID, pending.AttemptCount)
+ completed, err := repository.CompleteIfGeneration(db.GetDb(), pending.ID, pending.Generation)
require.NoError(t, err)
assert.False(t, completed, "the writer's receipt must not release an obligation someone else took")
@@ -135,7 +138,7 @@ func Test_CompleteIfGeneration_WhenGenerationMatches_ReleasesObligation(t *testi
pending, err := repository.InsertIfAbsent(db.GetDb(), uuid.New(), reference, 0)
require.NoError(t, err)
- completed, err := repository.CompleteIfGeneration(db.GetDb(), pending.ID, pending.AttemptCount)
+ completed, err := repository.CompleteIfGeneration(db.GetDb(), pending.ID, pending.Generation)
require.NoError(t, err)
assert.True(t, completed)
@@ -152,7 +155,7 @@ func Test_RescheduleIfGeneration_WhenGenerationIsStale_ChangesNothing(t *testing
require.NoError(t, err)
rescheduled, err := repository.RescheduleIfGeneration(
- db.GetDb(), pending.ID, pending.AttemptCount+1, time.Hour, "stale",
+ db.GetDb(), pending.ID, pending.Generation+1, time.Hour, "stale",
)
require.NoError(t, err)
@@ -171,7 +174,7 @@ func Test_ClaimDue_WhenNothingIsDue_ReturnsEmpty(t *testing.T) {
_, err := repository.InsertIfAbsent(db.GetDb(), uuid.New(), reference, time.Hour)
require.NoError(t, err)
- claimed, err := repository.ClaimDue(db.GetDb(), ClaimRequest{Limit: 10, AttemptTimeout: time.Minute})
+ claimed, err := repository.ClaimDue(db.GetDb(), ClaimRequest{Limit: 10, AttemptLease: time.Minute})
require.NoError(t, err)
assert.NotContains(t, referencesOf(claimed), reference)
@@ -183,15 +186,16 @@ func Test_ClaimDue_WhenRowIsDue_TakesItAndRaisesGeneration(t *testing.T) {
require.NoError(t, repository.InsertOrTake(db.GetDb(), []StoredFileReference{reference}))
- claimed, err := repository.ClaimDue(db.GetDb(), ClaimRequest{Limit: 100, AttemptTimeout: time.Minute})
+ claimed, err := repository.ClaimDue(db.GetDb(), ClaimRequest{Limit: 100, AttemptLease: time.Minute})
require.NoError(t, err)
require.Contains(t, referencesOf(claimed), reference)
found, err := repository.FindByReference(db.GetDb(), reference)
require.NoError(t, err)
require.NotNil(t, found)
+ assert.Equal(t, 1, found.Generation)
assert.Equal(t, 1, found.AttemptCount)
- assert.False(t, isDue(t, reference), "a claimed row is not due again until its attempt times out")
+ assert.False(t, isDue(t, reference), "a claimed row is not due again until its lease runs out")
}
func Test_ClaimDue_WhenWriteIsLive_SkipsIt(t *testing.T) {
@@ -204,7 +208,7 @@ func Test_ClaimDue_WhenWriteIsLive_SkipsIt(t *testing.T) {
require.NoError(t, err)
claimed, err := repository.ClaimDue(db.GetDb(), ClaimRequest{
- Limit: 100, AttemptTimeout: time.Minute, ExcludedIDs: []uuid.UUID{live.ID},
+ Limit: 100, AttemptLease: time.Minute, ExcludedIDs: []uuid.UUID{live.ID},
})
require.NoError(t, err)
@@ -240,7 +244,7 @@ func Test_ClaimDue_WhenAnotherTransactionHoldsTheRow_SkipsItInsteadOfWaiting(t *
<-blocked
start := time.Now()
- claimed, err := repository.ClaimDue(db.GetDb(), ClaimRequest{Limit: 100, AttemptTimeout: time.Minute})
+ claimed, err := repository.ClaimDue(db.GetDb(), ClaimRequest{Limit: 100, AttemptLease: time.Minute})
elapsed := time.Since(start)
require.NoError(t, err)
diff --git a/backend/internal/features/storages/files/store.go b/backend/internal/features/storages/files/store.go
index 28e93c8c..13d1d777 100644
--- a/backend/internal/features/storages/files/store.go
+++ b/backend/internal/features/storages/files/store.go
@@ -75,7 +75,7 @@ func (s *Store) WriteFile(
}
kept, err := s.repository.SetNotBeforeIfGeneration(
- db.GetDb(), pending.ID, pending.AttemptCount, s.timings.CommitWindow,
+ db.GetDb(), pending.ID, pending.Generation, s.timings.CommitWindow,
)
if err != nil {
return WriteReceipt{}, err
@@ -88,7 +88,7 @@ func (s *Store) WriteFile(
return WriteReceipt{
PendingDeletionID: pending.ID,
Reference: reference,
- Generation: pending.AttemptCount,
+ Generation: pending.Generation,
}, nil
}
@@ -150,7 +150,7 @@ func (s *Store) GetTimings() Timings {
// anyway. The write is over either way, so the failure is logged, not returned.
func (s *Store) makeEligibleNow(ctx context.Context, pending *PendingDeletion) {
if _, err := s.repository.SetNotBeforeIfGeneration(
- db.GetDb(), pending.ID, pending.AttemptCount, 0,
+ db.GetDb(), pending.ID, pending.Generation, 0,
); err != nil {
s.logger.ErrorContext(ctx, "failed to schedule cleanup for a failed write",
"storage_id", pending.StorageID,
diff --git a/backend/internal/features/storages/files/testing.go b/backend/internal/features/storages/files/testing.go
index 2f282b01..6e15270f 100644
--- a/backend/internal/features/storages/files/testing.go
+++ b/backend/internal/features/storages/files/testing.go
@@ -16,7 +16,7 @@ const drainDeadlineForTest = 30 * time.Second
func TimingsForTest() Timings {
return Timings{
CommitWindow: 2 * time.Second,
- AttemptTimeout: time.Second,
+ AttemptLease: time.Second,
RetryBaseDelay: 10 * time.Millisecond,
RetryMaxDelay: 50 * time.Millisecond,
WorkerTick: 10 * time.Millisecond,
diff --git a/backend/internal/features/storages/files/timings.go b/backend/internal/features/storages/files/timings.go
index 2829503b..6c969906 100644
--- a/backend/internal/features/storages/files/timings.go
+++ b/backend/internal/features/storages/files/timings.go
@@ -9,9 +9,9 @@ type Timings struct {
// CommitWindow is how long a file may stay recorded but unclaimed by any
// catalog transaction before cleanup assumes its publisher is never coming.
CommitWindow time.Duration
- // AttemptTimeout bounds one provider deletion and is also how far a claim
+ // AttemptLease bounds one provider deletion and is also how far a claim
// pushes not_before, so a stalled call cannot hold its row forever.
- AttemptTimeout time.Duration
+ AttemptLease time.Duration
RetryBaseDelay time.Duration
RetryMaxDelay time.Duration
WorkerTick time.Duration
@@ -21,7 +21,7 @@ type Timings struct {
func ProductionTimings() Timings {
return Timings{
CommitWindow: 15 * time.Minute,
- AttemptTimeout: 2 * time.Minute,
+ AttemptLease: 2 * time.Minute,
RetryBaseDelay: 30 * time.Second,
RetryMaxDelay: time.Hour,
WorkerTick: time.Minute,
diff --git a/backend/internal/features/storages/files/worker.go b/backend/internal/features/storages/files/worker.go
index df71406d..b04942a6 100644
--- a/backend/internal/features/storages/files/worker.go
+++ b/backend/internal/features/storages/files/worker.go
@@ -102,8 +102,9 @@ func (w *DeletionWorker) RunOnce(ctx context.Context) {
// DrainStorage empties one storage's obligations before the storage row, and the
// credentials with it, disappear. It returns the names it could not remove, which
-// is what the caller has to tell the user about. The budget bounds it because it
-// runs in the request that deletes the storage.
+// is what the caller has to tell the user about. The budget stops it starting more
+// files, because it runs in the request that deletes the storage; a call already
+// under way runs to the provider's own deadline.
func (w *DeletionWorker) DrainStorage(
ctx context.Context,
storageID uuid.UUID,
@@ -124,7 +125,7 @@ func (w *DeletionWorker) DrainStorage(
continue
}
- attemptCtx, cancel := context.WithTimeout(ctx, w.timings.AttemptTimeout)
+ attemptCtx, cancel := context.WithTimeout(ctx, w.timings.AttemptLease)
deleteErr := w.deleteFromProvider(attemptCtx, w.logger, obligation)
cancel()
@@ -142,9 +143,9 @@ func (w *DeletionWorker) claimDue() ([]PendingDeletion, error) {
err := w.store.withLiveWrites(func(excluded []uuid.UUID) error {
batch, claimErr := w.repository.ClaimDue(db.GetDb(), ClaimRequest{
- Limit: w.timings.ClaimBatchSize,
- AttemptTimeout: w.timings.AttemptTimeout,
- ExcludedIDs: excluded,
+ Limit: w.timings.ClaimBatchSize,
+ AttemptLease: w.timings.AttemptLease,
+ ExcludedIDs: excluded,
})
claimed = batch
@@ -157,13 +158,13 @@ func (w *DeletionWorker) claimDue() ([]PendingDeletion, error) {
func (w *DeletionWorker) attemptDeletion(ctx context.Context, passLogger *slog.Logger, pending PendingDeletion) {
attemptLogger := passLogger.With("storage_id", pending.StorageID, "file_name", pending.FileName)
- attemptCtx, cancel := context.WithTimeout(ctx, w.timings.AttemptTimeout)
+ attemptCtx, cancel := context.WithTimeout(ctx, w.timings.AttemptLease)
defer cancel()
err := w.deleteFromProvider(attemptCtx, attemptLogger, pending)
if err == nil {
released, completeErr := w.repository.CompleteIfGeneration(
- db.GetDb(), pending.ID, pending.AttemptCount,
+ db.GetDb(), pending.ID, pending.Generation,
)
if completeErr != nil {
attemptLogger.ErrorContext(ctx, "failed to release a completed file deletion", "error", completeErr)
@@ -186,7 +187,7 @@ func (w *DeletionWorker) attemptDeletion(ctx context.Context, passLogger *slog.L
retryDelay := w.retryDelay(pending.AttemptCount)
rescheduled, rescheduleErr := w.repository.RescheduleIfGeneration(
- db.GetDb(), pending.ID, pending.AttemptCount, retryDelay, sanitizeError(err),
+ db.GetDb(), pending.ID, pending.Generation, retryDelay, sanitizeError(err),
)
if rescheduleErr != nil {
attemptLogger.ErrorContext(ctx, "failed to reschedule a file deletion", "error", rescheduleErr)
diff --git a/backend/internal/features/storages/files/worker_test.go b/backend/internal/features/storages/files/worker_test.go
index faa89472..085d7e5d 100644
--- a/backend/internal/features/storages/files/worker_test.go
+++ b/backend/internal/features/storages/files/worker_test.go
@@ -187,6 +187,29 @@ func Test_RunOnce_WhenProviderFailsThenRecovers_RetriesUntilTheFileIsGone(t *tes
assert.Nil(t, pending)
}
+func Test_RunOnce_WhenDeletionWasRequestedRepeatedly_CountsOnlyTheAttempt(t *testing.T) {
+ provider := &fakeProvider{deleteErr: errors.New("storage is unavailable")}
+ locator := &fakeLocator{provider: provider}
+ store := newTestStore(locator)
+ worker := newTestWorker(t, store, locator)
+ reference := StoredFileReference{StorageID: createStorageRow(t), FileName: "backup-1"}
+
+ for range 3 {
+ require.NoError(t, inTransaction(t, func(tx *gorm.DB) error {
+ return store.RequestFileDeletions(t.Context(), tx, []StoredFileReference{reference})
+ }))
+ }
+
+ worker.RunOnce(t.Context())
+
+ pending, err := store.repository.FindByReference(db.GetDb(), reference)
+ require.NoError(t, err)
+ require.NotNil(t, pending)
+ assert.Equal(t, 3, pending.Generation, "each request takes the file from whoever held it")
+ assert.Equal(t, 1, pending.AttemptCount,
+ "three requests and one failed deletion is one attempt, so the retry waits the base delay")
+}
+
func Test_RunOnce_WhenAnotherCallerTakesTheFileMidAttempt_DoesNotReleaseTheirObligation(t *testing.T) {
provider := &fakeProvider{}
locator := &fakeLocator{provider: provider}
@@ -443,7 +466,7 @@ func Test_TimingsForTest_ShrinksEveryBoundReachingStoreAndWorker(t *testing.T) {
shrunk := TimingsForTest()
assert.Less(t, shrunk.CommitWindow, production.CommitWindow)
- assert.Less(t, shrunk.AttemptTimeout, production.AttemptTimeout)
+ assert.Less(t, shrunk.AttemptLease, production.AttemptLease)
assert.Less(t, shrunk.RetryBaseDelay, production.RetryBaseDelay)
assert.Less(t, shrunk.RetryMaxDelay, production.RetryMaxDelay)
assert.Less(t, shrunk.WorkerTick, production.WorkerTick)
diff --git a/backend/internal/features/storages/pending_deletion_drain.go b/backend/internal/features/storages/pending_deletion_drain.go
index 1a926f78..11cb7073 100644
--- a/backend/internal/features/storages/pending_deletion_drain.go
+++ b/backend/internal/features/storages/pending_deletion_drain.go
@@ -10,8 +10,10 @@ import (
"databasus-backend/internal/util/logger"
)
-// The drain runs in the request that deletes the storage, so an unreachable
-// provider costs the user this much waiting and no more.
+// The drain runs in the request that deletes the storage, so this is how long it
+// keeps starting files. The deadline is read between files and a provider holds
+// its own deadline on the call it already began, so an unreachable one costs this
+// plus one provider timeout.
const storageDeletionDrainBudget = 10 * time.Second
// A deleted storage takes its pending deletions with it, so this is the last
diff --git a/backend/migrations/20260914164105_split_pending_deletion_generation.sql b/backend/migrations/20260914164105_split_pending_deletion_generation.sql
new file mode 100644
index 00000000..bf84eb90
--- /dev/null
+++ b/backend/migrations/20260914164105_split_pending_deletion_generation.sql
@@ -0,0 +1,24 @@
+-- +goose Up
+-- +goose StatementBegin
+-- attempt_count carried two meanings: the retry counter the backoff reads and the
+-- generation a write receipt matches. Every deletion request raised it, so a file
+-- requested a few times began its first real attempt already deep in the backoff.
+-- Generation moves on its own now, and attempt_count counts only attempts.
+ALTER TABLE storage_pending_deletions
+ ADD COLUMN generation INTEGER NOT NULL DEFAULT 0;
+
+UPDATE storage_pending_deletions
+SET generation = attempt_count;
+
+ALTER TABLE storage_pending_deletions
+ ADD CONSTRAINT ck_storage_pending_deletions_generation_not_negative CHECK (generation >= 0);
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+ALTER TABLE storage_pending_deletions
+ DROP CONSTRAINT IF EXISTS ck_storage_pending_deletions_generation_not_negative;
+
+ALTER TABLE storage_pending_deletions
+ DROP COLUMN IF EXISTS generation;
+-- +goose StatementEnd
diff --git a/openspec/changes/ensure-storage-file-cleanup/design.md b/openspec/changes/ensure-storage-file-cleanup/design.md
index 973cb7ca..c64f005b 100644
--- a/openspec/changes/ensure-storage-file-cleanup/design.md
+++ b/openspec/changes/ensure-storage-file-cleanup/design.md
@@ -73,7 +73,8 @@ Add `storage_pending_deletions` with these columns:
| `id` | UUID row and receipt identity |
| `storage_id`, `file_name` | Logical file reference, unique as a pair |
| `not_before` | Earliest time the worker may act on this file |
-| `attempt_count` | Retry count, and the generation that invalidates a stale receipt |
+| `generation` | The value a write receipt matches, raised by anything that takes the file from its writer |
+| `attempt_count` | Deletion attempts, which is what the retry backoff grows with |
| `last_error` | Sanitized bounded failure text |
| `created_at`, `updated_at` | Ordering and operational diagnosis |
@@ -81,7 +82,7 @@ The table has a unique constraint on `(storage_id, file_name)`, a foreign key to
`ON DELETE CASCADE` is deliberate. The alternative, a restrictive foreign key, turns every teardown fixture and every workspace cascade into a raw database error and buys nothing: the storage row is what carries the credentials, so once it is gone the obligation is not executable anyway. Storage deletion instead drains what it can and records what it could not, which is covered below.
-`attempt_count` carries two meanings, so the migration and the repository both say so. It schedules retries, and it is the generation number that lets a write receipt detect that cleanup, or another caller, took the file away from its writer.
+`generation` and `attempt_count` move separately on purpose. A deletion request has to raise the generation, because that is what lets a write receipt detect that cleanup, or another caller, took the file away from its writer. It is not a deletion attempt, though, and letting it drive the backoff would start the first real attempt against a file that was requested a few times already deep in the retry curve.
Alternatives rejected:
@@ -93,7 +94,7 @@ Alternatives rejected:
`Store.WriteFile` generates the row ID, joins an in-process set of live writes, inserts the pending deletion row with `not_before = now() + commitWindow`, then calls the provider. Joining before the insert and starting the row ineligible close the same gap from two sides: a row the worker can see belongs to a writer that already joined, and a freshly inserted row is not due anyway. When the provider returns:
-- on success it refreshes `not_before` to `now() + commitWindow` conditionally on `attempt_count` still holding the value it started with, leaves the live set and returns a `WriteReceipt` carrying the row ID and that `attempt_count`;
+- on success it refreshes `not_before` to `now() + commitWindow` conditionally on `generation` still holding the value it started with, leaves the live set and returns a `WriteReceipt` carrying the row ID and that `generation`;
- on failure it sets `not_before = now()` under the same condition and only then leaves the live set, so the file becomes eligible exactly when its writer is gone;
- if either update matches no row, someone requested deletion of this file while the write was running, so it returns an error instead of a receipt and the caller never tries to publish.
@@ -101,7 +102,7 @@ The insert takes no action on conflict, and `WriteFile` then fails without sendi
The caller then does one of two things:
-- `ConfirmFileWrites(ctx, tx, receipts)` inside the transaction that makes the backup, WAL, history, manifest or metadata catalog state usable. It deletes each row by `id` conditionally on the receipt's `attempt_count` and fails the transaction unless every receipt matched exactly one row;
+- `ConfirmFileWrites(ctx, tx, receipts)` inside the transaction that makes the backup, WAL, history, manifest or metadata catalog state usable. It deletes each row by `id` conditionally on the receipt's `generation` and fails the transaction unless every receipt matched exactly one row;
- `RequestFileDeletions(ctx, tx, references)` inside the transaction that records failure, cancellation, retention removal or user deletion.
`commitWindow` is the only deadline in the design. It answers a single question: how long may a file sit recorded but unclaimed by any catalog transaction before cleanup assumes its publisher is never coming. Publication normally follows the upload within the same function, so the window is generous.
@@ -130,14 +131,16 @@ Alternatives rejected:
### Let the worker skip live writes instead of leasing them
-`DeletionWorker` holds the live-write mutex across its claim statement and passes the set into it: `UPDATE ... SET attempt_count = attempt_count + 1, not_before = now() + attemptTimeout WHERE id IN (SELECT id FROM storage_pending_deletions WHERE not_before <= now() AND id <> ALL(<live writes>) ORDER BY not_before LIMIT <batch> FOR UPDATE SKIP LOCKED) RETURNING *`. It commits the claim, then deletes remotely without holding a database lock. Both completion and rescheduling are conditional on the claimed `attempt_count`: success deletes the row, failure records a sanitized error and a jittered exponential `not_before` capped at a package maximum.
+`DeletionWorker` holds the live-write mutex across its claim statement and passes the set into it: `UPDATE ... SET generation = generation + 1, attempt_count = attempt_count + 1, not_before = now() + attemptLease WHERE id IN (SELECT id FROM storage_pending_deletions WHERE not_before <= now() AND id <> ALL(<live writes>) ORDER BY not_before LIMIT <batch> FOR UPDATE SKIP LOCKED) RETURNING *`. It commits the claim, then deletes remotely without holding a database lock. Both completion and rescheduling are conditional on the claimed `generation`: success deletes the row, failure records a sanitized error and a jittered exponential `not_before` capped at a package maximum and grown from `attempt_count`.
`SKIP LOCKED` matters because `RequestFileDeletions` runs inside a caller's catalog transaction, and task 8.4 makes database removal one transaction that records every FULL, INCR, WAL, history, manifest and metadata reference it owns. Without it, one such transaction would stall the whole claim batch rather than the rows it holds. Holding the mutex across that statement, rather than snapshotting the set before it, is what makes the exclusion sound. A writer joins the set before it inserts, so a writer that joins after the claim has not inserted its row yet and a writer that joined before it is in the exclusion list. Snapshotting first leaves a gap in between, and the gap is reachable: `RequestFileDeletions` can pull a live writer's row forward to `now()`, which a cascade does to a FULL or INCR file whose name the catalog already carries (`create_full_uc.go:40`). The mutex is safe to hold here because `FOR UPDATE SKIP LOCKED` means the statement never waits on a row lock, and no writer holds it across an insert.
Two properties make this enough, and both follow from one process:
- a row absent from the live set has no writer in this address space, and no other address space exists, so nothing is going to publish it;
-- the claiming update is a compare and swap on `attempt_count`, so a receipt issued before the claim can no longer confirm, and a slow attempt that outlived its `attemptTimeout` can neither reschedule nor complete over a newer one.
+- the claiming update is a compare and swap on `generation`, so a receipt issued before the claim can no longer confirm, and a slow attempt that outlived its `attemptLease` can neither reschedule nor complete over a newer one.
+
+`attemptLease` is how long a claimed row belongs to the pass that took it, not a bound on the provider call. Every provider gives its own deadline to a deletion and takes the passed context only for log correlation, so the lease has to stay longer than the slowest of those deadlines: a lease that runs out first lets the next pass claim a row whose attempt is still running, and the attempt that finishes second can no longer release it.
The live set is a mutex-guarded map of row IDs written by `Store.WriteFile` and read by the worker inside the same process. A writer joins before its insert and leaves only after its own database update has committed, so the worker can never observe a row as both eligible and unowned while its writer is still deciding the outcome.
@@ -170,7 +173,7 @@ Alternatives rejected:
### Accept a deletion request at any point in the write lifecycle
-`RequestFileDeletions` never fails because a writer is still running and never fails because the same reference was requested before. It inserts the row with `not_before = now()` when it is missing, because a file nobody is writing has no writer to wait for, and against an existing row it sets `not_before = now()` and increments `attempt_count`. That increment is what makes the request safe against a live writer: the writer's own completion update and its receipt both carry the earlier value, so the writer cannot extend the window and its caller cannot publish the file.
+`RequestFileDeletions` never fails because a writer is still running and never fails because the same reference was requested before. It inserts the row with `not_before = now()` when it is missing, because a file nobody is writing has no writer to wait for, and against an existing row it sets `not_before = now()` and raises `generation`. That raise is what makes the request safe against a live writer: the writer's own completion update and its receipt both carry the earlier value, so the writer cannot extend the window and its caller cannot publish the file.
The alternative shape, where the method rejects a request against a live writer, breaks a caller that already exists. `DeleteFull` cascade deletion runs while WAL archiving is active and is documented to catch a claim mid-upload (`.../wal_upload.go:275`). A rejection there would roll back the whole cascade transaction every time a segment happens to be uploading.
@@ -206,7 +209,7 @@ Alternatives rejected:
Storage deletion refuses while backup rows still reference the storage, which is the one question that has to leave the feature. It is answered through the existing registration mechanism at `backend/internal/features/storages/interfaces.go:48`, extended with a second method `GetStorageBackupReferences` rather than duplicated as a parallel registry, and implemented by the backup services that already register there. A counter that also reports backup references no longer matches the name `StorageDatabaseCounter`, so the interface, its registrar, the service field `storageDatabaseCounters` (`backend/internal/features/storages/service.go:22`) and the fixture `SetStorageDatabaseCountersForTest` (`backend/internal/features/storages/testing.go:13`) are renamed together.
-Pending deletions do not block anything. When a storage with no referencing backups is deleted, the service runs one drain pass over that storage's rows, bounded by both a row count and a wall-clock budget, then deletes the storage row, which cascades the rest away. The budget matters because the drain runs in the request path and the workspace listener repeats it for every storage in the workspace, so without it an unreachable provider would hold the user for the batch size times the provider timeout. The drain claims through the same path as the worker, so it excludes live writes for the same reason and leaves any file still being written to the cascade and the audit entry. Every pending deletion row still present for that storage after the drain goes into one audit-log entry through `AuditLogService.WriteAuditLog` (`backend/internal/features/audit_logs/service.go:24`) naming the storage, how many files are left behind and the first of their names up to a fixed limit, plus the same at warning level. The count tells the user the size of what was left, and the names give them something to act on, without putting thousands of lines into one audit entry. Reporting on what remains rather than on what the drain failed at also covers the rows the drain never saw, which are the ones the worker had already claimed and the ones its skip-locked claim passed over. Workspace deletion reaches storages through the listener at `backend/internal/features/storages/service.go:73` and takes the same path. `OnBeforeWorkspaceDeletion(workspaceID uuid.UUID) error` (`backend/internal/features/workspaces/interfaces/interfaces.go:6`) carries no context, and the drain gets a background one. Threading a context through three implementations buys one log field on a path that already outlives the request that started it.
+Pending deletions do not block anything. When a storage with no referencing backups is deleted, the service runs one drain pass over that storage's rows, bounded by both a row count and a wall-clock budget, then deletes the storage row, which cascades the rest away. The budget matters because the drain runs in the request path and the workspace listener repeats it for every storage in the workspace, so without it an unreachable provider would hold the user for the batch size times the provider timeout. The budget is read between files rather than inside one, so the worst case it buys is the budget plus one provider timeout. The drain claims through the same path as the worker, so it excludes live writes for the same reason and leaves any file still being written to the cascade and the audit entry. Every pending deletion row still present for that storage after the drain goes into one audit-log entry through `AuditLogService.WriteAuditLog` (`backend/internal/features/audit_logs/service.go:24`) naming the storage, how many files are left behind and the first of their names up to a fixed limit, plus the same at warning level. The count tells the user the size of what was left, and the names give them something to act on, without putting thousands of lines into one audit entry. Reporting on what remains rather than on what the drain failed at also covers the rows the drain never saw, which are the ones the worker had already claimed and the ones its skip-locked claim passed over. Workspace deletion reaches storages through the listener at `backend/internal/features/storages/service.go:73` and takes the same path. `OnBeforeWorkspaceDeletion(workspaceID uuid.UUID) error` (`backend/internal/features/workspaces/interfaces/interfaces.go:6`) carries no context, and the drain gets a background one. Threading a context through three implementations buys one log field on a path that already outlives the request that started it.
The user-visible outcome matches what a retry-until-abandoned design would eventually produce, without making the user wait out an abandonment age for it, and without the fourth lifecycle state and the restrictive foreign key that design needs.
@@ -234,7 +237,7 @@ Alternatives rejected:
## Risks / Trade-offs
- [A hung provider write blocks cleanup of its own file] -> The row stays in the live set until the call returns or the process stops, so the file is cleaned late rather than early. Nothing is deleted under a live writer, which is the property worth protecting.
-- [`attempt_count` carries two meanings] -> The migration comment, the repository method names and the design state that every event taking a file away from its writer increments it. A separate generation column would add a second number that always moves with the first.
+- [Two numbers move on the same row] -> `generation` is the compare-and-swap value every receipt matches and `attempt_count` is the retry counter. Folding them into one column was the first shape, and it made every deletion request look like a failed attempt, so a file requested a few times began its first real attempt near the retry cap.
- [A published backup could lose its file if publication is slower than `commitWindow`] -> Confirmation is conditional on the receipt generation, so the transaction fails instead of publishing a file that cleanup already claimed. The backup is then failed rather than silently broken.
- [Storage deletion now refuses while backup rows reference the storage] -> The refusal is actionable and the path out is the backup deletion flow this change makes reliable. The previous behavior removed the rows and made the objects unreachable forever.
- [An obligation against a permanently broken target retries forever] -> Nothing bounds the number of attempts, so a file whose provider never accepts a deletion is retried once per capped retry delay for as long as its storage exists. No storage, workspace or backup operation waits on it, and the summary's oldest-obligation age keeps it visible. What it does cost is queue position: the claim orders by `not_before`, so a backlog of permanently overdue rows on one storage sorts ahead of a row that just came due on another, and the second row waits about backlog divided by batch size passes. There is no per-storage fairness rule, because expressing one would need a window function that PostgreSQL will not accept alongside `FOR UPDATE SKIP LOCKED`, and the delay it would prevent is bounded, visible through the oldest-obligation age and measured in passes rather than in anything a user waits on. This is the price of rejecting the alternative that stops retrying past a bounded age, which needs a fourth lifecycle state and an age threshold to end.
diff --git a/openspec/changes/ensure-storage-file-cleanup/specs/storage-file-cleanup/spec.md b/openspec/changes/ensure-storage-file-cleanup/specs/storage-file-cleanup/spec.md
index 7544752e..ce3042eb 100644
--- a/openspec/changes/ensure-storage-file-cleanup/specs/storage-file-cleanup/spec.md
+++ b/openspec/changes/ensure-storage-file-cleanup/specs/storage-file-cleanup/spec.md
@@ -194,7 +194,7 @@ Every storage provider SHALL treat deletion as removal of all physical state der
### Requirement: Failed deletions remain retryable and observable
-The system SHALL retry failed deletions with exponential delay, a maximum delay and jitter. It SHALL persist the attempt count, the next attempt time and a sanitized bounded error message. Each worker pass SHALL have a stable `job_name` and a fresh `job_id`, and SHALL report how many obligations are pending, how many are overdue and the age of the oldest one. Every deletion attempt it logs SHALL identify the job, storage, file, attempt and next retry without exposing credentials.
+The system SHALL retry failed deletions with exponential delay, a maximum delay and jitter, grown from the number of deletion attempts alone. It SHALL persist that attempt count, the next attempt time and a sanitized bounded error message. Each worker pass SHALL have a stable `job_name` and a fresh `job_id`, and SHALL report how many obligations are pending, how many are overdue and the age of the oldest one. Every deletion attempt it logs SHALL identify the job, storage, file, attempt and next retry without exposing credentials.
#### Scenario: Storage outage persists
@@ -208,6 +208,12 @@ The system SHALL retry failed deletions with exponential delay, a maximum delay
- **THEN** the recorded error names the missing permission as the provider reported it
- **AND** the obligation stays pending so it succeeds once the permission is granted
+#### Scenario: A file is requested for deletion more than once before any attempt
+
+- **WHEN** several callers request deletion of the same file before the worker has attempted it
+- **THEN** the file carries one obligation
+- **AND** its first failed attempt waits the base retry delay, because a request is not an attempt
+
#### Scenario: Storage recovers
- **WHEN** a later deletion attempt succeeds after earlier failures
diff --git a/openspec/changes/ensure-storage-file-cleanup/tasks.md b/openspec/changes/ensure-storage-file-cleanup/tasks.md
index b5b8124e..f3e10301 100644
--- a/openspec/changes/ensure-storage-file-cleanup/tasks.md
+++ b/openspec/changes/ensure-storage-file-cleanup/tasks.md
@@ -1,9 +1,9 @@
## 1. Establish the pending deletion model
-- [x] 1.1 Add the `storage_pending_deletions` migration with `not_before`, `attempt_count`, `last_error`, the unique `(storage_id, file_name)` constraint, the `ON DELETE CASCADE` storage foreign key and one index on `not_before`, commenting the dual role of `attempt_count` as retry counter and receipt generation; verify `make migration-up-test` applies it and `make migration-down` reverses it.
+- [x] 1.1 Add the `storage_pending_deletions` migration with `not_before`, `generation`, `attempt_count`, `last_error`, the unique `(storage_id, file_name)` constraint, the `ON DELETE CASCADE` storage foreign key and one index on `not_before`; verify `make migration-up-test` applies it and `make migration-down` reverses it. The plan gave `attempt_count` both meanings and the first implementation followed it, which made every deletion request count as a failed attempt: a file requested a few times began its first real attempt near the retry cap. A follow-up migration splits the compare-and-swap value into `generation` and leaves `attempt_count` counting attempts.
- [x] 1.2 Add `StoredFileReference`, `WriteReceipt` and the narrow `FileWriter`, `FileRemover` and `StorageLocator` contracts plus `PendingDeletionRepository` under `backend/internal/features/storages/files` as `package storage_files`, split across `doc.go`, `store.go`, `repository.go`, `worker.go`, `model.go`, `dto.go`, `errors.go` and `timings.go` so the background loop and the transport types do not share a file with the store; verify with `make test` and confirm by inspection that the package imports neither its parent `storages` package nor any backup repository.
- [x] 1.3 Implement the repository operations: insert-if-absent, insert-or-take, generation-conditional completion, generation-conditional reschedule and a skip-locked batch claim excluding a caller-supplied set of live IDs; verify rollback, an insert that conflicts with an existing row, duplicate request, stale-generation and empty-batch cases with repository tests, and that a claim skips rows locked by an uncommitted deletion request instead of waiting for it.
-- [x] 1.4 Add the package timing constants for the commit window, the attempt timeout, the retry base and maximum delay and the worker tick, with a test constructor that shrinks them; verify the shrunk values reach every consumer so no test waits on production timings.
+- [x] 1.4 Add the package timing constants for the commit window, the attempt lease, the retry base and maximum delay and the worker tick, with a test constructor that shrinks them; verify the shrunk values reach every consumer so no test waits on production timings.
## 2. Route writes through the file store
@@ -15,7 +15,7 @@
## 3. Make deletion durable and terminating
- [x] 3.1 Implement one deletion-worker pass that takes the live-write mutex, reads the set under it and holds it until its claim statement returns, so no writer can join in between; that statement claims a bounded batch of due rows with skip-locked row selection ordered by `not_before`, excludes the set and increments the generation, commits the claim and only then performs remote I/O; verify a live write is never claimed even when it runs longer than the scan interval or when a cascading deletion request pulls its row forward to now, a file inherited from a stopped process is claimed on the first pass, a claim invalidates an outstanding receipt and an open deletion request does not block the batch.
-- [x] 3.2 Complete a successful deletion by removing the row and reschedule a failed one with capped exponential backoff, jitter, the incremented attempt count and a sanitized error, both conditional on the generation it claimed; verify transient failure, process restart, an attempt that outlived its timeout, an attempt whose row and storage were both removed by a storage deletion while it ran, and eventual success. Since `not_before` is compared against the database clock, drive these tests with the shrunk constants from task 1.4, a deterministic jitter source and fixtures that write `not_before` directly, not with a Go-side clock.
+- [x] 3.2 Complete a successful deletion by removing the row and reschedule a failed one with capped exponential backoff, jitter, the incremented attempt count and a sanitized error, both conditional on the generation it claimed; verify transient failure, process restart, an attempt that outlived its lease, an attempt whose row and storage were both removed by a storage deletion while it ran, and eventual success. Since `not_before` is compared against the database clock, drive these tests with the shrunk constants from task 1.4, a deterministic jitter source and fixtures that write `not_before` directly, not with a Go-side clock.
- [x] 3.3 Persist only bounded, sanitized deletion errors and emit attempt, retry and success logs carrying `job_name`, a fresh `job_id`, `storage_id`, `file_name` and `error` as key-value pairs, with the attempt count and the next retry time in the message text and the per-pass pending count, overdue count and oldest-obligation age in the summary message, following the value-in-message rule in `backend/AGENTS.md`; verify centralized redaction and truncation preserve these fields while removing credentials, storage configuration and signed URLs.
- [x] 3.4 Add a non-overlapping `Run` guard and a single-pass execution seam, then register graceful startup and shutdown with the application lifecycle in `backend/cmd/main.go`; verify a second `Run` panics, that overlapping ticks are skipped and that a cancelled context ends the loop only after the pass in flight has finished its provider call. The worker is started with a bare `go`, the same as every other background service in `cmd/main.go`, so nothing joins it at shutdown.