Self-hosted personal gallery application with online photo management and albums, supporting Live/Motion Photos, EXIF parsing, geolocation recognition, and an explore map.
TypeScript albumsexif-extractiongeocodinglive-photomotion-photophoto-galleryphoto-sharingwebgl-viewer
fix(cloud): cap lifecycle email delivery at three attempts
Timothy Yincd2c486363ce
diff --git a/apps/admin b/apps/adminindex d6dd36d6..2c962220 160000--- a/apps/admin+++ b/apps/admin@@ -1 +1 @@-Subproject commit d6dd36d61f9935864cf5b1e971bc0a8ef6210ffc+Subproject commit 2c962220f521a270131fde6860361db0db433ea2diff --git a/apps/cloud/server/plugins/account-lifecycle.ts b/apps/cloud/server/plugins/account-lifecycle.tsindex 2133a99a..c0e8af52 100644--- a/apps/cloud/server/plugins/account-lifecycle.ts+++ b/apps/cloud/server/plugins/account-lifecycle.ts@@ -108,7 +108,7 @@ export default defineNitroPlugin((app) => {outbox: async () => {await db.prepare(- "DELETE FROM platform_mail_outbox WHERE state IN('sent','cancelled') AND available_at<?",+ "DELETE FROM platform_mail_outbox WHERE state IN('sent','cancelled','failed') AND available_at<?",).bind(now - 30 * 86400000).run()diff --git a/packages/platform/migrations/0021_mail_retry_budget.sql b/packages/platform/migrations/0021_mail_retry_budget.sqlnew file mode 100644index 00000000..2511cd77--- /dev/null+++ b/packages/platform/migrations/0021_mail_retry_budget.sql@@ -0,0 +1,27 @@+-- Delivery attempts include the initial send. Exhausted messages remain inspectable.+CREATE TABLE platform_mail_outbox_next (+ id TEXT PRIMARY KEY,+ user_id INTEGER NOT NULL,+ kind TEXT NOT NULL,+ cycle TEXT NOT NULL,+ to_email TEXT NOT NULL,+ payload_json TEXT NOT NULL,+ available_at INTEGER NOT NULL,+ attempts INTEGER NOT NULL DEFAULT 0,+ state TEXT NOT NULL DEFAULT 'pending' CHECK(state IN('pending','sent','cancelled','failed')),+ lease_token TEXT,+ lease_until INTEGER,+ error TEXT,+ UNIQUE(user_id,kind,cycle)+);+INSERT INTO platform_mail_outbox_next+ SELECT id,user_id,kind,cycle,to_email,payload_json,available_at,+ attempts+CASE WHEN state='sent' THEN 1 ELSE 0 END,+ CASE WHEN state='pending' AND attempts>=3 THEN 'failed' ELSE state END,+ CASE WHEN state='pending' AND attempts>=3 THEN NULL ELSE lease_token END,+ CASE WHEN state='pending' AND attempts>=3 THEN NULL ELSE lease_until END,+ CASE WHEN state='pending' AND attempts>=3 THEN 'email_delivery_exhausted' ELSE error END+ FROM platform_mail_outbox;+DROP TABLE platform_mail_outbox;+ALTER TABLE platform_mail_outbox_next RENAME TO platform_mail_outbox;+CREATE INDEX platform_mail_pending ON platform_mail_outbox(state,available_at);diff --git a/packages/platform/src/account-lifecycle.ts b/packages/platform/src/account-lifecycle.tsindex c128f48f..d3ed1f55 100644--- a/packages/platform/src/account-lifecycle.ts+++ b/packages/platform/src/account-lifecycle.ts@@ -428,10 +428,17 @@ export async function deliverLifecycleMail(now = Date.now(),) {const s = await lifecycleSettings(db)+ // A worker may die after claiming the final attempt; reclaiming must not send a fourth time.+ await db+ .prepare(+ "UPDATE platform_mail_outbox SET state='failed',error='email_delivery_exhausted',lease_token=NULL,lease_until=NULL WHERE state='pending' AND attempts>=3 AND (lease_until IS NULL OR lease_until<?)",+ )+ .bind(now)+ .run()const rows = await allRows<any>(db.prepare(- "SELECT * FROM platform_mail_outbox WHERE state='pending' AND available_at<=? AND (lease_until IS NULL OR lease_until<?) ORDER BY available_at,id LIMIT 10",+ "SELECT * FROM platform_mail_outbox WHERE state='pending' AND attempts<3 AND available_at<=? AND (lease_until IS NULL OR lease_until<?) ORDER BY available_at,id LIMIT 10",).bind(now, now),)@@ -439,10 +446,10 @@ export async function deliverLifecycleMail(const token = crypto.randomUUID(),claimed = await db.prepare(- "UPDATE platform_mail_outbox SET lease_token=?,lease_until=? WHERE id=? AND state='pending' AND (lease_until IS NULL OR lease_until<?) RETURNING id",+ "UPDATE platform_mail_outbox SET lease_token=?,lease_until=?,attempts=attempts+1 WHERE id=? AND state='pending' AND attempts<3 AND available_at<=? AND (lease_until IS NULL OR lease_until<?) RETURNING attempts",)- .bind(token, now + 120000, row.id, now)- .first()+ .bind(token, now + 120000, row.id, now, now)+ .first<{ attempts: number }>()if (!claimed) continuetry {const p = JSON.parse(row.payload_json),@@ -491,10 +498,16 @@ export async function deliverLifecycleMail(} catch {await db.prepare(- "UPDATE platform_mail_outbox SET attempts=attempts+1,available_at=?,error='email_delivery_failed',lease_token=NULL,lease_until=NULL WHERE id=? AND lease_token=?",+ 'UPDATE platform_mail_outbox SET state=?,available_at=?,error=?,lease_token=NULL,lease_until=NULL WHERE id=? AND lease_token=?',).bind(- now + Math.min(3600000, 60000 * 2 ** Math.min(row.attempts, 6)),+ claimed.attempts >= 3 ? 'failed' : 'pending',+ claimed.attempts >= 3+ ? now+ : now + 60000 * 2 ** (claimed.attempts - 1),+ claimed.attempts >= 3+ ? 'email_delivery_exhausted'+ : 'email_delivery_failed',row.id,token,)diff --git a/tests/account-lifecycle-mail.test.ts b/tests/account-lifecycle-mail.test.tsindex ce760210..fd9c1ede 100644--- a/tests/account-lifecycle-mail.test.ts+++ b/tests/account-lifecycle-mail.test.ts@@ -498,6 +498,50 @@ test('Templates escape content, validate headers, preserve auth requirements andnow + 59999,)assert.equal(calls, 1)+ await deliverLifecycleMail(+ db,+ failing,+ 'https://app.chronoframe.art',+ now + 60000,+ )+ assert.equal(calls, 2)+ assert.equal(+ (+ await db+ .prepare('SELECT available_at FROM platform_mail_outbox')+ .first()+ ).available_at,+ now + 180000,+ )+ await deliverLifecycleMail(+ db,+ failing,+ 'https://app.chronoframe.art',+ now + 179999,+ )+ assert.equal(calls, 2)+ await deliverLifecycleMail(+ db,+ failing,+ 'https://app.chronoframe.art',+ now + 180000,+ )+ assert.equal(calls, 3)+ const exhausted = await db+ .prepare('SELECT attempts,state,error FROM platform_mail_outbox')+ .first()+ assert.deepEqual(exhausted, {+ attempts: 3,+ state: 'failed',+ error: 'email_delivery_exhausted',+ })+ await deliverLifecycleMail(+ db,+ failing,+ 'https://app.chronoframe.art',+ now + DAY,+ )+ assert.equal(calls, 3)const auth = createAccountAuth(db,{@@ -1151,3 +1195,274 @@ test('Versioned S3 cleanup batches 1000 scoped versions and surfaces partial delfail = trueawait assert.rejects(s3.purgeTenant(8), /batch removal incomplete/)})+test('Delivery attempts are reserved atomically, capped across worker crashes and counted on success', async () => {+ const { mf, db } = await fixture()+ try {+ const now = Date.now()+ for (let id = 1; id <= 4; id++)+ await db+ .prepare(+ 'INSERT INTO platform_users(id,email,created_at,last_login_at,email_verified) VALUES(?,?,?,?,1)',+ )+ .bind(id, `budget${id}@example.com`, now, now)+ .run()+ await markAccount(+ db,+ 1,+ 'Concurrent delivery attempt budget',+ 'manual',+ null,+ now,+ )+ let calls = 0+ const failure = {+ send: async () => {+ calls+++ throw new Error('failed')+ },+ }+ for (const delta of [0, 60000, 180000])+ await Promise.all(+ Array.from({ length: 4 }, () =>+ deliverLifecycleMail(+ db,+ failure,+ 'https://app.chronoframe.art',+ now + delta,+ ),+ ),+ )+ assert.equal(calls, 3)+ assert.equal(+ (+ await db+ .prepare('SELECT state FROM platform_mail_outbox WHERE user_id=1')+ .first()+ ).state,+ 'failed',+ )+ await markAccount(+ db,+ 2,+ 'Interrupted final delivery attempt',+ 'manual',+ null,+ now,+ )+ await db+ .prepare(+ "UPDATE platform_mail_outbox SET attempts=3,lease_token='interrupted',lease_until=? WHERE user_id=2",+ )+ .bind(now + 120000)+ .run()+ await deliverLifecycleMail(+ db,+ failure,+ 'https://app.chronoframe.art',+ now + 60000,+ )+ assert.equal(+ (+ await db+ .prepare('SELECT state FROM platform_mail_outbox WHERE user_id=2')+ .first()+ ).state,+ 'pending',+ )+ await deliverLifecycleMail(+ db,+ failure,+ 'https://app.chronoframe.art',+ now + 120001,+ )+ assert.equal(+ (+ await db+ .prepare('SELECT state FROM platform_mail_outbox WHERE user_id=2')+ .first()+ ).state,+ 'failed',+ )+ assert.equal(calls, 3)+ await markAccount(+ db,+ 3,+ 'Successful final delivery attempt',+ 'manual',+ null,+ now,+ )+ let successCalls = 0+ const eventually = {+ send: async () => {+ if (++successCalls < 3) throw new Error('failed')+ return { messageId: 'delivered' }+ },+ }+ for (const delta of [0, 60000, 180000])+ await deliverLifecycleMail(+ db,+ eventually,+ 'https://app.chronoframe.art',+ now + delta,+ )+ assert.deepEqual(+ await db+ .prepare(+ 'SELECT state,attempts FROM platform_mail_outbox WHERE user_id=3',+ )+ .first(),+ { state: 'sent', attempts: 3 },+ )+ await deliverLifecycleMail(+ db,+ eventually,+ 'https://app.chronoframe.art',+ now + DAY,+ )+ assert.equal(successCalls, 3)+ await markAccount(+ db,+ 4,+ 'Stale concurrent worker respects backoff',+ 'manual',+ null,+ now,+ )+ let reads = 0,+ claims = 0,+ raceCalls = 0+ let releaseReads!: () => void, releaseFailure!: () => void+ const bothRead = new Promise<void>((resolve) => {+ releaseReads = resolve+ }),+ firstFailed = new Promise<void>((resolve) => {+ releaseFailure = resolve+ })+ const wrap = (statement: any, sql: string): any => ({+ bind(...values: any[]) {+ return wrap(statement.bind(...values), sql)+ },+ async all() {+ const result = await statement.all()+ if (+ sql.startsWith(+ "SELECT * FROM platform_mail_outbox WHERE state='pending'",+ )+ ) {+ if (++reads === 2) releaseReads()+ await bothRead+ }+ return result+ },+ async first() {+ if (sql.includes('attempts=attempts+1') && ++claims === 2)+ await firstFailed+ return statement.first()+ },+ async run() {+ const result = await statement.run()+ if (sql.includes('SET state=?,available_at=?')) releaseFailure()+ return result+ },+ })+ const raced = {+ prepare: (sql: string) => wrap(db.prepare(sql), sql),+ batch: db.batch.bind(db),+ }+ const raceFailure = {+ send: async () => {+ raceCalls+++ throw new Error('failed')+ },+ }+ await Promise.all([+ deliverLifecycleMail(+ raced as any,+ raceFailure,+ 'https://app.chronoframe.art',+ now,+ ),+ deliverLifecycleMail(+ raced as any,+ raceFailure,+ 'https://app.chronoframe.art',+ now,+ ),+ ])+ assert.equal(raceCalls, 1)+ assert.deepEqual(+ await db+ .prepare(+ 'SELECT attempts,available_at FROM platform_mail_outbox WHERE user_id=4',+ )+ .first(),+ { attempts: 1, available_at: now + 60000 },+ )+ } finally {+ await mf.dispose()+ }+})+test('Retry-budget migration preserves delivery history and terminates already-exhausted messages', async () => {+ const mf = new Miniflare(+ convertV4MiniflareOptions({+ modules: true,+ compatibilityDate: '2026-10-26',+ script: 'export default {fetch(){return new Response("ok")}}',+ d1Databases: ['DB'],+ }),+ )+ try {+ const db = await mf.getD1Database('DB')+ for (const name of (+ await readdir(+ new URL('../packages/platform/migrations/', import.meta.url),+ )+ )+ .filter((f) => f.endsWith('.sql'))+ .sort()) {+ if (name.startsWith('0021'))+ await db+ .prepare(+ "INSERT INTO platform_mail_outbox(id,user_id,kind,cycle,to_email,payload_json,available_at,attempts,state) VALUES('old-pending',1,'account-deleted','a','[email protected]','{}',0,5,'pending'),('old-sent',1,'account-deleted','b','[email protected]','{}',0,2,'sent'),('old-retry',1,'account-deleted','c','[email protected]','{}',0,2,'pending')",+ )+ .run()+ for (const sql of (+ await readFile(+ new URL('../packages/platform/migrations/' + name, import.meta.url),+ 'utf8',+ )+ )+ .replace(/^--.*$/gm, '')+ .split(';')+ .filter((s) => s.trim()))+ await db.prepare(sql).run()+ }+ assert.deepEqual(+ await db+ .prepare(+ "SELECT state,attempts,error FROM platform_mail_outbox WHERE id='old-pending'",+ )+ .first(),+ { state: 'failed', attempts: 5, error: 'email_delivery_exhausted' },+ )+ assert.deepEqual(+ await db+ .prepare(+ "SELECT state,attempts FROM platform_mail_outbox WHERE id='old-sent'",+ )+ .first(),+ { state: 'sent', attempts: 3 },+ )+ assert.deepEqual(+ await db+ .prepare(+ "SELECT state,attempts FROM platform_mail_outbox WHERE id='old-retry'",+ )+ .first(),+ { state: 'pending', attempts: 2 },+ )+ } finally {+ await mf.dispose()+ }+})