Skip to content

Self-hosted personal gallery application with online photo management and albums, supporting Live/Motion Photos, EXIF parsing, geolocation recognition, and an explore map.

fix(cloud): cap lifecycle email delivery at three attempts

Timothy Yincd2c486363ce
diff --git a/apps/admin b/apps/admin
index d6dd36d6..2c962220 160000
--- a/apps/admin
+++ b/apps/admin
@@ -1 +1 @@
-Subproject commit d6dd36d61f9935864cf5b1e971bc0a8ef6210ffc
+Subproject commit 2c962220f521a270131fde6860361db0db433ea2
diff --git a/apps/cloud/server/plugins/account-lifecycle.ts b/apps/cloud/server/plugins/account-lifecycle.ts
index 2133a99a..c0e8af52 100644
--- a/apps/cloud/server/plugins/account-lifecycle.ts
+++ b/apps/cloud/server/plugins/account-lifecycle.ts
@@ -108,7 +108,7 @@ export default defineNitroPlugin((app) => {
outbox: async () => {
await db
.prepare(
- "DELETE FROM platform_mail_outbox WHERE state IN('sent','cancelled') AND available_at<?",
+ "DELETE FROM platform_mail_outbox WHERE state IN('sent','cancelled','failed') AND available_at<?",
)
.bind(now - 30 * 86400000)
.run()
diff --git a/packages/platform/migrations/0021_mail_retry_budget.sql b/packages/platform/migrations/0021_mail_retry_budget.sql
new file mode 100644
index 00000000..2511cd77
--- /dev/null
+++ b/packages/platform/migrations/0021_mail_retry_budget.sql
@@ -0,0 +1,27 @@
+-- Delivery attempts include the initial send. Exhausted messages remain inspectable.
+CREATE TABLE platform_mail_outbox_next (
+ id TEXT PRIMARY KEY,
+ user_id INTEGER NOT NULL,
+ kind TEXT NOT NULL,
+ cycle TEXT NOT NULL,
+ to_email TEXT NOT NULL,
+ payload_json TEXT NOT NULL,
+ available_at INTEGER NOT NULL,
+ attempts INTEGER NOT NULL DEFAULT 0,
+ state TEXT NOT NULL DEFAULT 'pending' CHECK(state IN('pending','sent','cancelled','failed')),
+ lease_token TEXT,
+ lease_until INTEGER,
+ error TEXT,
+ UNIQUE(user_id,kind,cycle)
+);
+INSERT INTO platform_mail_outbox_next
+ SELECT id,user_id,kind,cycle,to_email,payload_json,available_at,
+ attempts+CASE WHEN state='sent' THEN 1 ELSE 0 END,
+ CASE WHEN state='pending' AND attempts>=3 THEN 'failed' ELSE state END,
+ CASE WHEN state='pending' AND attempts>=3 THEN NULL ELSE lease_token END,
+ CASE WHEN state='pending' AND attempts>=3 THEN NULL ELSE lease_until END,
+ CASE WHEN state='pending' AND attempts>=3 THEN 'email_delivery_exhausted' ELSE error END
+ FROM platform_mail_outbox;
+DROP TABLE platform_mail_outbox;
+ALTER TABLE platform_mail_outbox_next RENAME TO platform_mail_outbox;
+CREATE INDEX platform_mail_pending ON platform_mail_outbox(state,available_at);
diff --git a/packages/platform/src/account-lifecycle.ts b/packages/platform/src/account-lifecycle.ts
index c128f48f..d3ed1f55 100644
--- a/packages/platform/src/account-lifecycle.ts
+++ b/packages/platform/src/account-lifecycle.ts
@@ -428,10 +428,17 @@ export async function deliverLifecycleMail(
now = Date.now(),
) {
const s = await lifecycleSettings(db)
+ // A worker may die after claiming the final attempt; reclaiming must not send a fourth time.
+ await db
+ .prepare(
+ "UPDATE platform_mail_outbox SET state='failed',error='email_delivery_exhausted',lease_token=NULL,lease_until=NULL WHERE state='pending' AND attempts>=3 AND (lease_until IS NULL OR lease_until<?)",
+ )
+ .bind(now)
+ .run()
const rows = await allRows<any>(
db
.prepare(
- "SELECT * FROM platform_mail_outbox WHERE state='pending' AND available_at<=? AND (lease_until IS NULL OR lease_until<?) ORDER BY available_at,id LIMIT 10",
+ "SELECT * FROM platform_mail_outbox WHERE state='pending' AND attempts<3 AND available_at<=? AND (lease_until IS NULL OR lease_until<?) ORDER BY available_at,id LIMIT 10",
)
.bind(now, now),
)
@@ -439,10 +446,10 @@ export async function deliverLifecycleMail(
const token = crypto.randomUUID(),
claimed = await db
.prepare(
- "UPDATE platform_mail_outbox SET lease_token=?,lease_until=? WHERE id=? AND state='pending' AND (lease_until IS NULL OR lease_until<?) RETURNING id",
+ "UPDATE platform_mail_outbox SET lease_token=?,lease_until=?,attempts=attempts+1 WHERE id=? AND state='pending' AND attempts<3 AND available_at<=? AND (lease_until IS NULL OR lease_until<?) RETURNING attempts",
)
- .bind(token, now + 120000, row.id, now)
- .first()
+ .bind(token, now + 120000, row.id, now, now)
+ .first<{ attempts: number }>()
if (!claimed) continue
try {
const p = JSON.parse(row.payload_json),
@@ -491,10 +498,16 @@ export async function deliverLifecycleMail(
} catch {
await db
.prepare(
- "UPDATE platform_mail_outbox SET attempts=attempts+1,available_at=?,error='email_delivery_failed',lease_token=NULL,lease_until=NULL WHERE id=? AND lease_token=?",
+ 'UPDATE platform_mail_outbox SET state=?,available_at=?,error=?,lease_token=NULL,lease_until=NULL WHERE id=? AND lease_token=?',
)
.bind(
- now + Math.min(3600000, 60000 * 2 ** Math.min(row.attempts, 6)),
+ claimed.attempts >= 3 ? 'failed' : 'pending',
+ claimed.attempts >= 3
+ ? now
+ : now + 60000 * 2 ** (claimed.attempts - 1),
+ claimed.attempts >= 3
+ ? 'email_delivery_exhausted'
+ : 'email_delivery_failed',
row.id,
token,
)
diff --git a/tests/account-lifecycle-mail.test.ts b/tests/account-lifecycle-mail.test.ts
index ce760210..fd9c1ede 100644
--- a/tests/account-lifecycle-mail.test.ts
+++ b/tests/account-lifecycle-mail.test.ts
@@ -498,6 +498,50 @@ test('Templates escape content, validate headers, preserve auth requirements and
now + 59999,
)
assert.equal(calls, 1)
+ await deliverLifecycleMail(
+ db,
+ failing,
+ 'https://app.chronoframe.art',
+ now + 60000,
+ )
+ assert.equal(calls, 2)
+ assert.equal(
+ (
+ await db
+ .prepare('SELECT available_at FROM platform_mail_outbox')
+ .first()
+ ).available_at,
+ now + 180000,
+ )
+ await deliverLifecycleMail(
+ db,
+ failing,
+ 'https://app.chronoframe.art',
+ now + 179999,
+ )
+ assert.equal(calls, 2)
+ await deliverLifecycleMail(
+ db,
+ failing,
+ 'https://app.chronoframe.art',
+ now + 180000,
+ )
+ assert.equal(calls, 3)
+ const exhausted = await db
+ .prepare('SELECT attempts,state,error FROM platform_mail_outbox')
+ .first()
+ assert.deepEqual(exhausted, {
+ attempts: 3,
+ state: 'failed',
+ error: 'email_delivery_exhausted',
+ })
+ await deliverLifecycleMail(
+ db,
+ failing,
+ 'https://app.chronoframe.art',
+ now + DAY,
+ )
+ assert.equal(calls, 3)
const auth = createAccountAuth(
db,
{
@@ -1151,3 +1195,274 @@ test('Versioned S3 cleanup batches 1000 scoped versions and surfaces partial del
fail = true
await assert.rejects(s3.purgeTenant(8), /batch removal incomplete/)
})
+test('Delivery attempts are reserved atomically, capped across worker crashes and counted on success', async () => {
+ const { mf, db } = await fixture()
+ try {
+ const now = Date.now()
+ for (let id = 1; id <= 4; id++)
+ await db
+ .prepare(
+ 'INSERT INTO platform_users(id,email,created_at,last_login_at,email_verified) VALUES(?,?,?,?,1)',
+ )
+ .bind(id, `budget${id}@example.com`, now, now)
+ .run()
+ await markAccount(
+ db,
+ 1,
+ 'Concurrent delivery attempt budget',
+ 'manual',
+ null,
+ now,
+ )
+ let calls = 0
+ const failure = {
+ send: async () => {
+ calls++
+ throw new Error('failed')
+ },
+ }
+ for (const delta of [0, 60000, 180000])
+ await Promise.all(
+ Array.from({ length: 4 }, () =>
+ deliverLifecycleMail(
+ db,
+ failure,
+ 'https://app.chronoframe.art',
+ now + delta,
+ ),
+ ),
+ )
+ assert.equal(calls, 3)
+ assert.equal(
+ (
+ await db
+ .prepare('SELECT state FROM platform_mail_outbox WHERE user_id=1')
+ .first()
+ ).state,
+ 'failed',
+ )
+ await markAccount(
+ db,
+ 2,
+ 'Interrupted final delivery attempt',
+ 'manual',
+ null,
+ now,
+ )
+ await db
+ .prepare(
+ "UPDATE platform_mail_outbox SET attempts=3,lease_token='interrupted',lease_until=? WHERE user_id=2",
+ )
+ .bind(now + 120000)
+ .run()
+ await deliverLifecycleMail(
+ db,
+ failure,
+ 'https://app.chronoframe.art',
+ now + 60000,
+ )
+ assert.equal(
+ (
+ await db
+ .prepare('SELECT state FROM platform_mail_outbox WHERE user_id=2')
+ .first()
+ ).state,
+ 'pending',
+ )
+ await deliverLifecycleMail(
+ db,
+ failure,
+ 'https://app.chronoframe.art',
+ now + 120001,
+ )
+ assert.equal(
+ (
+ await db
+ .prepare('SELECT state FROM platform_mail_outbox WHERE user_id=2')
+ .first()
+ ).state,
+ 'failed',
+ )
+ assert.equal(calls, 3)
+ await markAccount(
+ db,
+ 3,
+ 'Successful final delivery attempt',
+ 'manual',
+ null,
+ now,
+ )
+ let successCalls = 0
+ const eventually = {
+ send: async () => {
+ if (++successCalls < 3) throw new Error('failed')
+ return { messageId: 'delivered' }
+ },
+ }
+ for (const delta of [0, 60000, 180000])
+ await deliverLifecycleMail(
+ db,
+ eventually,
+ 'https://app.chronoframe.art',
+ now + delta,
+ )
+ assert.deepEqual(
+ await db
+ .prepare(
+ 'SELECT state,attempts FROM platform_mail_outbox WHERE user_id=3',
+ )
+ .first(),
+ { state: 'sent', attempts: 3 },
+ )
+ await deliverLifecycleMail(
+ db,
+ eventually,
+ 'https://app.chronoframe.art',
+ now + DAY,
+ )
+ assert.equal(successCalls, 3)
+ await markAccount(
+ db,
+ 4,
+ 'Stale concurrent worker respects backoff',
+ 'manual',
+ null,
+ now,
+ )
+ let reads = 0,
+ claims = 0,
+ raceCalls = 0
+ let releaseReads!: () => void, releaseFailure!: () => void
+ const bothRead = new Promise<void>((resolve) => {
+ releaseReads = resolve
+ }),
+ firstFailed = new Promise<void>((resolve) => {
+ releaseFailure = resolve
+ })
+ const wrap = (statement: any, sql: string): any => ({
+ bind(...values: any[]) {
+ return wrap(statement.bind(...values), sql)
+ },
+ async all() {
+ const result = await statement.all()
+ if (
+ sql.startsWith(
+ "SELECT * FROM platform_mail_outbox WHERE state='pending'",
+ )
+ ) {
+ if (++reads === 2) releaseReads()
+ await bothRead
+ }
+ return result
+ },
+ async first() {
+ if (sql.includes('attempts=attempts+1') && ++claims === 2)
+ await firstFailed
+ return statement.first()
+ },
+ async run() {
+ const result = await statement.run()
+ if (sql.includes('SET state=?,available_at=?')) releaseFailure()
+ return result
+ },
+ })
+ const raced = {
+ prepare: (sql: string) => wrap(db.prepare(sql), sql),
+ batch: db.batch.bind(db),
+ }
+ const raceFailure = {
+ send: async () => {
+ raceCalls++
+ throw new Error('failed')
+ },
+ }
+ await Promise.all([
+ deliverLifecycleMail(
+ raced as any,
+ raceFailure,
+ 'https://app.chronoframe.art',
+ now,
+ ),
+ deliverLifecycleMail(
+ raced as any,
+ raceFailure,
+ 'https://app.chronoframe.art',
+ now,
+ ),
+ ])
+ assert.equal(raceCalls, 1)
+ assert.deepEqual(
+ await db
+ .prepare(
+ 'SELECT attempts,available_at FROM platform_mail_outbox WHERE user_id=4',
+ )
+ .first(),
+ { attempts: 1, available_at: now + 60000 },
+ )
+ } finally {
+ await mf.dispose()
+ }
+})
+test('Retry-budget migration preserves delivery history and terminates already-exhausted messages', async () => {
+ const mf = new Miniflare(
+ convertV4MiniflareOptions({
+ modules: true,
+ compatibilityDate: '2026-10-26',
+ script: 'export default {fetch(){return new Response("ok")}}',
+ d1Databases: ['DB'],
+ }),
+ )
+ try {
+ const db = await mf.getD1Database('DB')
+ for (const name of (
+ await readdir(
+ new URL('../packages/platform/migrations/', import.meta.url),
+ )
+ )
+ .filter((f) => f.endsWith('.sql'))
+ .sort()) {
+ if (name.startsWith('0021'))
+ await db
+ .prepare(
+ "INSERT INTO platform_mail_outbox(id,user_id,kind,cycle,to_email,payload_json,available_at,attempts,state) VALUES('old-pending',1,'account-deleted','a','[email protected]','{}',0,5,'pending'),('old-sent',1,'account-deleted','b','[email protected]','{}',0,2,'sent'),('old-retry',1,'account-deleted','c','[email protected]','{}',0,2,'pending')",
+ )
+ .run()
+ for (const sql of (
+ await readFile(
+ new URL('../packages/platform/migrations/' + name, import.meta.url),
+ 'utf8',
+ )
+ )
+ .replace(/^--.*$/gm, '')
+ .split(';')
+ .filter((s) => s.trim()))
+ await db.prepare(sql).run()
+ }
+ assert.deepEqual(
+ await db
+ .prepare(
+ "SELECT state,attempts,error FROM platform_mail_outbox WHERE id='old-pending'",
+ )
+ .first(),
+ { state: 'failed', attempts: 5, error: 'email_delivery_exhausted' },
+ )
+ assert.deepEqual(
+ await db
+ .prepare(
+ "SELECT state,attempts FROM platform_mail_outbox WHERE id='old-sent'",
+ )
+ .first(),
+ { state: 'sent', attempts: 3 },
+ )
+ assert.deepEqual(
+ await db
+ .prepare(
+ "SELECT state,attempts FROM platform_mail_outbox WHERE id='old-retry'",
+ )
+ .first(),
+ { state: 'pending', attempts: 2 },
+ )
+ } finally {
+ await mf.dispose()
+ }
+})