Open-source status page, incident response and uptime monitoring
TypeScript alertingincident-managementmcpmcp-servermonitoringmonitoring-as-codeobservabilityon-callopen-sourcesrestatus-pagestatuspagesynthetic-monitoringterraformtinybirdtursouptimeuptime-checkeruptime-monitor
fix: prevent Resend 409 idempotency conflicts on subscriber notifications (#2397)
Maximilian Kaske295e5a72f52c
diff --git a/packages/emails/src/client.test.ts b/packages/emails/src/client.test.tsindex 59aa8292f..aceb5291e 100644--- a/packages/emails/src/client.test.ts+++ b/packages/emails/src/client.test.ts@@ -39,6 +39,11 @@ function baseReq(const ok = { data: { data: [] }, error: null } as any;// biome-ignore lint/suspicious/noExplicitAny: simulated Resend application errorconst fail = { data: null, error: { name: "application_error" } } as any;+// biome-ignore lint/suspicious/noExplicitAny: simulated Resend 409 conflict+const idempotencyConflict = {+ data: null,+ error: { name: "invalid_idempotent_request" },+} as any;describe("EmailClient.sendStatusReportUpdate - idempotency & chunking", () => {let client: EmailClient;@@ -115,4 +120,19 @@ describe("EmailClient.sendStatusReportUpdate - idempotency & chunking", () => {"status-report-update:7:0",]);});++ test("does not retry a deterministic idempotency conflict", async () => {+ batchSend.restore();+ batchSend = stub(client.client.batch, "send", () =>+ Promise.resolve(idempotencyConflict),+ );++ await client.sendStatusReportUpdate(+ baseReq({ idempotencyKey: "status-report-update:7" }),+ );++ // a 409 invalid_idempotent_request replays identically on every attempt;+ // retrying only re-logs the same error.+ assertSpyCalls(batchSend, 1);+ });});diff --git a/packages/emails/src/client.tsx b/packages/emails/src/client.tsxindex f0b7da0df..721671f86 100644--- a/packages/emails/src/client.tsx+++ b/packages/emails/src/client.tsx@@ -26,6 +26,19 @@ export function statusReportSubject(req: {: req.reportTitle;}+// Deterministic Resend rejections: retrying the identical request can never+// succeed (e.g. 409 invalid_idempotent_request when a key is reused with a+// different body), it only burns the backoff and re-logs the error.+const NON_RETRYABLE_RESEND_ERRORS = new Set<string>([+ "invalid_idempotent_request",+ "invalid_idempotency_key",+ "validation_error",+]);++function isRetryableSendError(error: { name: string }): boolean {+ return !NON_RETRYABLE_RESEND_ERRORS.has(error.name);+}+// split an array into chunks of a given size.function chunk<T>(array: T[], size: number): T[][] {const result: T[][] = [];@@ -226,6 +239,7 @@ export class EmailClient {Effect.retry({times: 3,schedule: Schedule.exponential(this.retryBackoff),+ while: isRetryableSendError,}),);await Effect.runPromise(sendEmail).catch(console.error);@@ -421,6 +435,7 @@ export class EmailClient {Effect.retry({times: 3,schedule: Schedule.exponential(this.retryBackoff),+ while: isRetryableSendError,}),);await Effect.runPromise(sendEmail).catch(console.error);diff --git a/packages/subscriptions/src/channels/email.test.ts b/packages/subscriptions/src/channels/email.test.tsindex bda440661..7dbf54d55 100644--- a/packages/subscriptions/src/channels/email.test.ts+++ b/packages/subscriptions/src/channels/email.test.ts@@ -155,7 +155,7 @@ describe("sendEmailNotifications", () => {await sendEmailNotifications([sub], makeUpdate({ updateId: 77 }));const [args] = sendStatusReportUpdateMock.calls[0].args;- expect(args.idempotencyKey).toBe("status-report-update:77");+ expect(args.idempotencyKey).toMatch(/^status-report-update:77:[0-9a-z]+$/);});test("falls back to a page-update key when there is no update id (maintenance)", async () => {@@ -166,6 +166,50 @@ describe("sendEmailNotifications", () => {);const [args] = sendStatusReportUpdateMock.calls[0].args;- expect(args.idempotencyKey).toBe("page-update:17:maintenance");+ expect(args.idempotencyKey).toMatch(+ /^page-update:17:maintenance:[0-9a-z]+$/,+ );+ });++ test("keeps the same key for an identical re-dispatch", async () => {+ const update = makeUpdate({ updateId: 77, date: "2026-07-16T11:00:00Z" });+ await sendEmailNotifications([makeSub()], update);+ await sendEmailNotifications([makeSub()], update);++ const keys = sendStatusReportUpdateMock.calls.map(+ (c) => c.args[0].idempotencyKey,+ );+ expect(keys[0]).toBe(keys[1]);+ });++ test("changes the key when the subscriber list changes", async () => {+ const update = makeUpdate({ updateId: 77, date: "2026-07-16T11:00:00Z" });+ await sendEmailNotifications([makeSub()], update);+ await sendEmailNotifications(+ [makeSub(), makeSub({ id: 2, email: "[email protected]", token: "token-b" })],+ update,+ );++ const keys = sendStatusReportUpdateMock.calls.map(+ (c) => c.args[0].idempotencyKey,+ );+ expect(keys[0]).not.toBe(keys[1]);+ });++ test("changes the key when the update content changes", async () => {+ const date = "2026-07-16T11:00:00Z";+ await sendEmailNotifications(+ [makeSub()],+ makeUpdate({ updateId: 77, date, message: "We are investigating." }),+ );+ await sendEmailNotifications(+ [makeSub()],+ makeUpdate({ updateId: 77, date, message: "Root cause identified." }),+ );++ const keys = sendStatusReportUpdateMock.calls.map(+ (c) => c.args[0].idempotencyKey,+ );+ expect(keys[0]).not.toBe(keys[1]);});});diff --git a/packages/subscriptions/src/channels/email.ts b/packages/subscriptions/src/channels/email.tsindex 6e2cc6bac..79510a97b 100644--- a/packages/subscriptions/src/channels/email.ts+++ b/packages/subscriptions/src/channels/email.ts@@ -33,6 +33,19 @@ function idempotencyKeyFor(pageUpdate: PageUpdate): string {: `page-update:${pageUpdate.id}:${pageUpdate.status}`;}+// FNV-1a, Edge-safe (no node:crypto). Resend 409s when a key is reused within+// 24h with a different body (e.g. the subscriber list changed between two+// dispatches of the same update), so the key must vary with the payload while+// staying identical across retries of the same send.+function fingerprint(input: string): string {+ let hash = 0x811c9dc5;+ for (let i = 0; i < input.length; i++) {+ hash ^= input.charCodeAt(i);+ hash = Math.imul(hash, 0x01000193);+ }+ return (hash >>> 0).toString(36);+}+function hasEmailAndToken(sub: Subscription,): sub is Subscription & { email: string; token: string } {@@ -71,6 +84,20 @@ export async function sendEmailNotifications(const firstSub = validSubscriptions[0];+ const payloadHash = fingerprint(+ JSON.stringify([+ validSubscriptions.map((sub) => [sub.email, sub.token]),+ firstSub.pageName,+ firstSub.pageSlug,+ firstSub.customDomain ?? null,+ pageUpdate.title,+ pageUpdate.status,+ pageUpdate.message,+ pageUpdate.date,+ pageUpdate.pageComponents,+ ]),+ );+const client = getEmailClient();await client.sendStatusReportUpdate({subscribers: validSubscriptions.map((sub) => ({@@ -85,6 +112,6 @@ export async function sendEmailNotifications(message: pageUpdate.message,date: pageUpdate.date,pageComponents: pageUpdate.pageComponents,- idempotencyKey: idempotencyKeyFor(pageUpdate),+ idempotencyKey: `${idempotencyKeyFor(pageUpdate)}:${payloadHash}`,});}diff --git a/packages/subscriptions/src/dispatcher.ts b/packages/subscriptions/src/dispatcher.tsindex f687e5664..4186c8be7 100644--- a/packages/subscriptions/src/dispatcher.ts+++ b/packages/subscriptions/src/dispatcher.ts@@ -155,6 +155,9 @@ export async function dispatchPageUpdate(pageUpdate: PageUpdate) {isNotNull(pageSubscriber.acceptedAt),isNull(pageSubscriber.unsubscribedAt),),+ // Deterministic order so the email idempotency fingerprint is stable+ // across dispatches; id breaks same-second createdAt ties.+ orderBy: (subs, { asc }) => [asc(subs.createdAt), asc(subs.id)],with: {components: true,},