Skip to content

Open-source status page, incident response and uptime monitoring

fix: prevent Resend 409 idempotency conflicts on subscriber notifications (#2397)

Maximilian Kaske295e5a72f52c
diff --git a/packages/emails/src/client.test.ts b/packages/emails/src/client.test.ts
index 59aa8292f..aceb5291e 100644
--- a/packages/emails/src/client.test.ts
+++ b/packages/emails/src/client.test.ts
@@ -39,6 +39,11 @@ function baseReq(
const ok = { data: { data: [] }, error: null } as any;
// biome-ignore lint/suspicious/noExplicitAny: simulated Resend application error
const fail = { data: null, error: { name: "application_error" } } as any;
+// biome-ignore lint/suspicious/noExplicitAny: simulated Resend 409 conflict
+const idempotencyConflict = {
+ data: null,
+ error: { name: "invalid_idempotent_request" },
+} as any;
describe("EmailClient.sendStatusReportUpdate - idempotency & chunking", () => {
let client: EmailClient;
@@ -115,4 +120,19 @@ describe("EmailClient.sendStatusReportUpdate - idempotency & chunking", () => {
"status-report-update:7:0",
]);
});
+
+ test("does not retry a deterministic idempotency conflict", async () => {
+ batchSend.restore();
+ batchSend = stub(client.client.batch, "send", () =>
+ Promise.resolve(idempotencyConflict),
+ );
+
+ await client.sendStatusReportUpdate(
+ baseReq({ idempotencyKey: "status-report-update:7" }),
+ );
+
+ // a 409 invalid_idempotent_request replays identically on every attempt;
+ // retrying only re-logs the same error.
+ assertSpyCalls(batchSend, 1);
+ });
});
diff --git a/packages/emails/src/client.tsx b/packages/emails/src/client.tsx
index f0b7da0df..721671f86 100644
--- a/packages/emails/src/client.tsx
+++ b/packages/emails/src/client.tsx
@@ -26,6 +26,19 @@ export function statusReportSubject(req: {
: req.reportTitle;
}
+// Deterministic Resend rejections: retrying the identical request can never
+// succeed (e.g. 409 invalid_idempotent_request when a key is reused with a
+// different body), it only burns the backoff and re-logs the error.
+const NON_RETRYABLE_RESEND_ERRORS = new Set<string>([
+ "invalid_idempotent_request",
+ "invalid_idempotency_key",
+ "validation_error",
+]);
+
+function isRetryableSendError(error: { name: string }): boolean {
+ return !NON_RETRYABLE_RESEND_ERRORS.has(error.name);
+}
+
// split an array into chunks of a given size.
function chunk<T>(array: T[], size: number): T[][] {
const result: T[][] = [];
@@ -226,6 +239,7 @@ export class EmailClient {
Effect.retry({
times: 3,
schedule: Schedule.exponential(this.retryBackoff),
+ while: isRetryableSendError,
}),
);
await Effect.runPromise(sendEmail).catch(console.error);
@@ -421,6 +435,7 @@ export class EmailClient {
Effect.retry({
times: 3,
schedule: Schedule.exponential(this.retryBackoff),
+ while: isRetryableSendError,
}),
);
await Effect.runPromise(sendEmail).catch(console.error);
diff --git a/packages/subscriptions/src/channels/email.test.ts b/packages/subscriptions/src/channels/email.test.ts
index bda440661..7dbf54d55 100644
--- a/packages/subscriptions/src/channels/email.test.ts
+++ b/packages/subscriptions/src/channels/email.test.ts
@@ -155,7 +155,7 @@ describe("sendEmailNotifications", () => {
await sendEmailNotifications([sub], makeUpdate({ updateId: 77 }));
const [args] = sendStatusReportUpdateMock.calls[0].args;
- expect(args.idempotencyKey).toBe("status-report-update:77");
+ expect(args.idempotencyKey).toMatch(/^status-report-update:77:[0-9a-z]+$/);
});
test("falls back to a page-update key when there is no update id (maintenance)", async () => {
@@ -166,6 +166,50 @@ describe("sendEmailNotifications", () => {
);
const [args] = sendStatusReportUpdateMock.calls[0].args;
- expect(args.idempotencyKey).toBe("page-update:17:maintenance");
+ expect(args.idempotencyKey).toMatch(
+ /^page-update:17:maintenance:[0-9a-z]+$/,
+ );
+ });
+
+ test("keeps the same key for an identical re-dispatch", async () => {
+ const update = makeUpdate({ updateId: 77, date: "2026-07-16T11:00:00Z" });
+ await sendEmailNotifications([makeSub()], update);
+ await sendEmailNotifications([makeSub()], update);
+
+ const keys = sendStatusReportUpdateMock.calls.map(
+ (c) => c.args[0].idempotencyKey,
+ );
+ expect(keys[0]).toBe(keys[1]);
+ });
+
+ test("changes the key when the subscriber list changes", async () => {
+ const update = makeUpdate({ updateId: 77, date: "2026-07-16T11:00:00Z" });
+ await sendEmailNotifications([makeSub()], update);
+ await sendEmailNotifications(
+ [makeSub(), makeSub({ id: 2, email: "[email protected]", token: "token-b" })],
+ update,
+ );
+
+ const keys = sendStatusReportUpdateMock.calls.map(
+ (c) => c.args[0].idempotencyKey,
+ );
+ expect(keys[0]).not.toBe(keys[1]);
+ });
+
+ test("changes the key when the update content changes", async () => {
+ const date = "2026-07-16T11:00:00Z";
+ await sendEmailNotifications(
+ [makeSub()],
+ makeUpdate({ updateId: 77, date, message: "We are investigating." }),
+ );
+ await sendEmailNotifications(
+ [makeSub()],
+ makeUpdate({ updateId: 77, date, message: "Root cause identified." }),
+ );
+
+ const keys = sendStatusReportUpdateMock.calls.map(
+ (c) => c.args[0].idempotencyKey,
+ );
+ expect(keys[0]).not.toBe(keys[1]);
});
});
diff --git a/packages/subscriptions/src/channels/email.ts b/packages/subscriptions/src/channels/email.ts
index 6e2cc6bac..79510a97b 100644
--- a/packages/subscriptions/src/channels/email.ts
+++ b/packages/subscriptions/src/channels/email.ts
@@ -33,6 +33,19 @@ function idempotencyKeyFor(pageUpdate: PageUpdate): string {
: `page-update:${pageUpdate.id}:${pageUpdate.status}`;
}
+// FNV-1a, Edge-safe (no node:crypto). Resend 409s when a key is reused within
+// 24h with a different body (e.g. the subscriber list changed between two
+// dispatches of the same update), so the key must vary with the payload while
+// staying identical across retries of the same send.
+function fingerprint(input: string): string {
+ let hash = 0x811c9dc5;
+ for (let i = 0; i < input.length; i++) {
+ hash ^= input.charCodeAt(i);
+ hash = Math.imul(hash, 0x01000193);
+ }
+ return (hash >>> 0).toString(36);
+}
+
function hasEmailAndToken(
sub: Subscription,
): sub is Subscription & { email: string; token: string } {
@@ -71,6 +84,20 @@ export async function sendEmailNotifications(
const firstSub = validSubscriptions[0];
+ const payloadHash = fingerprint(
+ JSON.stringify([
+ validSubscriptions.map((sub) => [sub.email, sub.token]),
+ firstSub.pageName,
+ firstSub.pageSlug,
+ firstSub.customDomain ?? null,
+ pageUpdate.title,
+ pageUpdate.status,
+ pageUpdate.message,
+ pageUpdate.date,
+ pageUpdate.pageComponents,
+ ]),
+ );
+
const client = getEmailClient();
await client.sendStatusReportUpdate({
subscribers: validSubscriptions.map((sub) => ({
@@ -85,6 +112,6 @@ export async function sendEmailNotifications(
message: pageUpdate.message,
date: pageUpdate.date,
pageComponents: pageUpdate.pageComponents,
- idempotencyKey: idempotencyKeyFor(pageUpdate),
+ idempotencyKey: `${idempotencyKeyFor(pageUpdate)}:${payloadHash}`,
});
}
diff --git a/packages/subscriptions/src/dispatcher.ts b/packages/subscriptions/src/dispatcher.ts
index f687e5664..4186c8be7 100644
--- a/packages/subscriptions/src/dispatcher.ts
+++ b/packages/subscriptions/src/dispatcher.ts
@@ -155,6 +155,9 @@ export async function dispatchPageUpdate(pageUpdate: PageUpdate) {
isNotNull(pageSubscriber.acceptedAt),
isNull(pageSubscriber.unsubscribedAt),
),
+ // Deterministic order so the email idempotency fingerprint is stable
+ // across dispatches; id breaks same-second createdAt ties.
+ orderBy: (subs, { asc }) => [asc(subs.createdAt), asc(subs.id)],
with: {
components: true,
},