Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.
TypeScript acmeagentsai-agentscertificate-managementdevsecopsenvironment-variableskubernetesopen-sourcepampkipostgresprivate-caprivileged-access-managementsecret-managementsecret-managersecretssecrets-managementsecuritysecurity-toolsvault
fix retry horizon
Thiago Araujo da Silva875d1d0310de
diff --git a/backend/CLAUDE.md b/backend/CLAUDE.mdindex feabd0a9fe..1f07821c55 100644--- a/backend/CLAUDE.md+++ b/backend/CLAUDE.md@@ -681,7 +681,13 @@ the transaction.**The row owns the retry state machine** (`attempts`, `nextRetryAt`, backoff, terminal `failed`) and theoutbox owns the lag and exhausted metrics. `consumer.handle` owns delivery and what "delivered" means,-reporting per row: `Delivered` / `Retry` / `Failed` plus `progress`.+reporting per row: `Delivered` / `Retry` / `Failed` plus `progress`. Backoff is exponential with jitter+from a 30s base, and `MAX_OUTBOX_ATTEMPTS` is sized so the last attempt lands about an hour after the+first: a `failed` row is a notification the customer never receives and nothing replays it, so the+horizon has to outlast a realistic outage of the endpoint being delivered to. The stale sweeper applies+the same backoff rather than handing rows straight back, so a consumer that hangs every time cannot hold+a worker slot for the full threshold on repeat. To replay `failed` rows by hand, flip them to+`status = 'retry', attempts = 0, nextRetryAt = now()`; the next relay tick picks them up.**A claim is a lease, and `drain` keeps it alive.** The sweeper treats any `processing` row whose`lockedAt` is older than `STALE_CLAIM_THRESHOLD_MS` as a dead worker and hands it back out. A batch can@@ -689,6 +695,12 @@ legitimately outlive that (one slow webhook endpoint timing out on every row ofrefreshes `lockedAt` on the claimed ids every quarter of the threshold while `handle` runs. Without itthe sweeper re-queues rows a live worker is still delivering and the customer is notified twice.+**Discovery is scoped to the consumers this process has registered.** `findDueFlushKeys` takes+`eventOutboxRegistry.names()`, and that filter is load-bearing: rows for a name nobody here can drain (a+renamed consumer, a rolling deploy) stay `pending` forever, are never pruned, and sort first on every+tick, so without it they would fill the discovery limit and hide every real key behind them. They now+wait harmlessly and show up on the oldest-pending gauge instead.+**A consumer must not let one row's failure escape `handle`.** The outbox retries the whole batch when`handle` throws, because it cannot know which rows already delivered. A consumer that processes rows oneat a time catches per row and reports `Retry` for that row alone; see `alert-event-consumer.ts`. The@@ -720,9 +732,15 @@ undelivered row, trading a rare reorder for one failing event stalling everythinnotifications that is the wrong trade, so do not promise callers strict ordering.**Delivery is at-least-once.** A send that succeeds and whose `commitResults` then fails is redelivered.-Consumers narrow the blast radius with `progress` (alerting stores the channels that already succeeded-and skips them on the retry), and an emitter that needs to dedupe its own retries passes an-`idempotencyKey`.+The commit is retried a few times in-process first, since by then the consumer has already sent and the+alternative is the sweeper re-notifying ten minutes later. Consumers narrow what is left with `progress`+(alerting stores the channels that already succeeded and skips them on the retry), and an emitter that+needs to dedupe its own retries passes an `idempotencyKey`.++**`claimBatch` sorts what `RETURNING` gives it.** The subquery picks rows in id order, but+`UPDATE ... RETURNING` hands them back in whatever order it touched them (seen for real: 8, 7, 6), and+`handle` is promised id order. The `e2e-test/event-outbox.spec.ts` suite runs these queries against+Postgres because that is the only place this class of bug shows up.**Watch `infisical.event_outbox.oldest_pending_age`.** It is the canonical canary, the one signal thatcatches a dead relay, a wedged consumer and a stuck claim alike. Alarm when it exceeds a small multiplediff --git a/backend/e2e-test/event-outbox.spec.ts b/backend/e2e-test/event-outbox.spec.tsnew file mode 100644index 0000000000..3b70750e5d--- /dev/null+++ b/backend/e2e-test/event-outbox.spec.ts@@ -0,0 +1,196 @@+import { randomUUID } from "node:crypto";++import { Knex } from "knex";+import { afterAll, afterEach, beforeAll, describe, expect, test } from "vitest";++import { TableName } from "@app/db/schemas";+import { eventOutboxDALFactory, TOutboxInsertRow } from "@app/services/event-outbox/event-outbox-dal";+import { EventOutboxStatus, MAX_OUTBOX_ATTEMPTS } from "@app/services/event-outbox/event-outbox-types";++declare const testDb: Knex;++// The unit tests assert query shape. Only Postgres can show that the partial-index conflict target+// compiles, that two claimers never take the same row, and that the retry clock is computed in SQL.+const dal = eventOutboxDALFactory(testDb as never);++// Unique per run so a crashed run's leftovers can't leak into the next one's assertions.+const CONSUMER = `e2e-${randomUUID().slice(0, 8)}`;+const ORG_ID = randomUUID();++const KEY = { consumer: CONSUMER, resourceType: "e2e.resource", resourceId: "res-1" };++const makeRow = (overrides?: Partial<TOutboxInsertRow>): TOutboxInsertRow => ({+ consumer: CONSUMER,+ eventType: "e2e.resource.happened",+ resourceType: KEY.resourceType,+ resourceId: KEY.resourceId,+ orgId: ORG_ID,+ payload: { targetIds: ["t-1"] },+ occurredAt: new Date(),+ ...overrides+});++const insert = (rows: TOutboxInsertRow[]) => testDb.transaction((tx) => dal.insertEvents(rows, tx));++const rowsFor = (consumer = CONSUMER) =>+ testDb(TableName.EventOutbox).where({ consumer }).orderBy("id", "asc").select("*");++const purge = () => testDb(TableName.EventOutbox).where("consumer", "like", "e2e-%").del();++describe("event outbox (postgres)", () => {+ beforeAll(purge);+ afterEach(purge);+ afterAll(purge);++ test("insertEvents dedupes on (consumer, idempotencyKey) and lets rows without a key repeat", async () => {+ await insert([makeRow({ idempotencyKey: "k-1" }), makeRow(), makeRow()]);+ await insert([makeRow({ idempotencyKey: "k-1" })]);+ await insert([makeRow({ idempotencyKey: "k-1", consumer: `${CONSUMER}-other` })]);++ const mine = await rowsFor();+ expect(mine).toHaveLength(3);+ expect(mine.filter((row) => row.idempotencyKey === "k-1")).toHaveLength(1);+ expect(mine[0].status).toBe(EventOutboxStatus.Pending);+ expect(mine[0].attempts).toBe(0);++ expect(await rowsFor(`${CONSUMER}-other`)).toHaveLength(1);+ });++ test("claimBatch hands each row to exactly one concurrent claimer, in id order", async () => {+ await insert(Array.from({ length: 6 }, () => makeRow()));++ const [a, b] = await Promise.all([dal.claimBatch(KEY, 3), dal.claimBatch(KEY, 3)]);++ const idsA = a.map((row) => String(row.id));+ const idsB = b.map((row) => String(row.id));+ expect(new Set([...idsA, ...idsB]).size).toBe(idsA.length + idsB.length);+ expect(idsA.length + idsB.length).toBe(6);+ expect([...idsA].sort()).toEqual(idsA);+ expect([...idsB].sort()).toEqual(idsB);++ const stored = await rowsFor();+ expect(stored.every((row) => row.status === EventOutboxStatus.Processing && row.lockedAt)).toBe(true);+ expect(await dal.claimBatch(KEY, 10)).toEqual([]);+ });++ test("claimBatch skips a row still inside its backoff window", async () => {+ await insert([makeRow(), makeRow()]);+ const [first, second] = await rowsFor();+ await testDb(TableName.EventOutbox)+ .where("id", first.id)+ .update({ status: EventOutboxStatus.Retry, nextRetryAt: new Date(Date.now() + 60 * 60_000) });++ const claimed = await dal.claimBatch(KEY, 10);++ expect(claimed.map((row) => String(row.id))).toEqual([String(second.id)]);+ });++ test("commitResults applies every outcome and computes the retry clock in the database", async () => {+ await insert([makeRow(), makeRow(), makeRow()]);+ const [d, r, f] = await dal.claimBatch(KEY, 10);++ await dal.commitResults({+ delivered: [{ ids: [String(d.id)], progress: { deliveredChannelIds: ["c-1"] } }],+ retriable: [{ ids: [String(r.id)], nextRetryDelayMs: 30 * 60_000, error: "slack 502" }],+ failed: [{ ids: [String(f.id)], error: "bad payload" }]+ });++ const [delivered, retry, failed] = await rowsFor();+ expect(delivered.status).toBe(EventOutboxStatus.Delivered);+ expect(delivered.lockedAt).toBeNull();+ expect(delivered.progress).toEqual({ deliveredChannelIds: ["c-1"] });++ expect(retry.status).toBe(EventOutboxStatus.Retry);+ expect(retry.attempts).toBe(1);+ expect(retry.lastError).toBe("slack 502");+ expect(retry.lockedAt).toBeNull();+ expect(new Date(retry.nextRetryAt).getTime()).toBeGreaterThan(Date.now() + 29 * 60_000);++ expect(failed.status).toBe(EventOutboxStatus.Failed);+ expect(failed.attempts).toBe(1);+ expect(failed.lastError).toBe("bad payload");++ expect(await dal.findDueFlushKeys(10, [CONSUMER])).toEqual([]);+ });++ test("findDueFlushKeys groups due rows per resource and ignores consumers not asked for", async () => {+ await insert([+ makeRow(),+ makeRow(),+ makeRow({ resourceId: "res-2" }),+ makeRow({ consumer: `${CONSUMER}-other`, resourceId: "res-3" })+ ]);++ const keys = await dal.findDueFlushKeys(10, [CONSUMER]);++ expect(keys).toHaveLength(2);+ expect(keys.map((key) => key.resourceId).sort()).toEqual(["res-1", "res-2"]);+ expect(keys.every((key) => key.consumer === CONSUMER)).toBe(true);+ });++ test("recoverStaleClaims backs a live-looking claim off and fails an exhausted one", async () => {+ await insert([makeRow(), makeRow(), makeRow()]);+ const [, stale, exhausted] = await dal.claimBatch(KEY, 10);+ const longAgo = new Date(Date.now() - 60 * 60_000);+ await testDb(TableName.EventOutbox).where("id", stale.id).update({ lockedAt: longAgo });+ await testDb(TableName.EventOutbox)+ .where("id", exhausted.id)+ .update({ lockedAt: longAgo, attempts: MAX_OUTBOX_ATTEMPTS - 1 });++ const outcome = await dal.recoverStaleClaims(10 * 60_000, MAX_OUTBOX_ATTEMPTS);++ expect(outcome).toEqual({ retried: 1, failed: 1 });+ const [untouched, retried, failed] = await rowsFor();+ expect(untouched.status).toBe(EventOutboxStatus.Processing);+ expect(retried.status).toBe(EventOutboxStatus.Retry);+ expect(retried.attempts).toBe(1);+ expect(retried.lockedAt).toBeNull();+ // Not handed straight back: a consumer that hangs every time must not hold a worker slot on repeat.+ expect(new Date(retried.nextRetryAt).getTime()).toBeGreaterThan(Date.now() + 20_000);+ expect(failed.status).toBe(EventOutboxStatus.Failed);+ expect(failed.attempts).toBe(MAX_OUTBOX_ATTEMPTS);+ });++ test("extendClaims refreshes only rows still processing", async () => {+ await insert([makeRow(), makeRow()]);+ const [claimed, released] = await dal.claimBatch(KEY, 10);+ const longAgo = new Date(Date.now() - 60 * 60_000);+ await testDb(TableName.EventOutbox).whereIn("id", [claimed.id, released.id]).update({ lockedAt: longAgo });+ await dal.commitResults({ delivered: [{ ids: [String(released.id)] }], retriable: [], failed: [] });++ await dal.extendClaims([String(claimed.id), String(released.id)]);++ const [a, b] = await rowsFor();+ expect(new Date(a.lockedAt as Date).getTime()).toBeGreaterThan(Date.now() - 10_000);+ expect(b.lockedAt).toBeNull();+ });++ test("deleteTerminalOlderThan prunes only terminal rows past their retention, in bounded batches", async () => {+ const old = new Date(Date.now() - 48 * 60 * 60_000);+ await testDb(TableName.EventOutbox).insert([+ { ...makeRow(), payload: JSON.stringify({}), status: EventOutboxStatus.Delivered, updatedAt: old },+ { ...makeRow(), payload: JSON.stringify({}), status: EventOutboxStatus.Delivered, updatedAt: old },+ { ...makeRow(), payload: JSON.stringify({}), status: EventOutboxStatus.Failed, updatedAt: old },+ { ...makeRow(), payload: JSON.stringify({}), status: EventOutboxStatus.Pending, updatedAt: old },+ { ...makeRow(), payload: JSON.stringify({}), status: EventOutboxStatus.Delivered }+ ]);++ const firstBatch = await dal.deleteTerminalOlderThan({+ deliveredBefore: new Date(Date.now() - 24 * 60 * 60_000),+ failedBefore: new Date(Date.now() - 30 * 24 * 60 * 60_000),+ batchSize: 1+ });+ const rest = await dal.deleteTerminalOlderThan({+ deliveredBefore: new Date(Date.now() - 24 * 60 * 60_000),+ failedBefore: new Date(Date.now() - 30 * 24 * 60 * 60_000),+ batchSize: 100+ });++ expect(firstBatch).toBe(1);+ expect(rest).toBe(1);+ const remaining = await rowsFor();+ expect(remaining.map((row) => row.status).sort()).toEqual(+ [EventOutboxStatus.Delivered, EventOutboxStatus.Failed, EventOutboxStatus.Pending].sort()+ );+ });+});diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.tsindex 4348a21083..5ae1a122d3 100644--- a/backend/src/server/routes/index.ts+++ b/backend/src/server/routes/index.ts@@ -1079,6 +1079,7 @@ export const registerRoutes = async (const eventOutboxQueue = eventOutboxQueueFactory({queueService,cronJob,+ eventOutboxRegistry,eventOutboxDAL,eventOutboxService});diff --git a/backend/src/services/event-outbox/event-outbox-dal.test.ts b/backend/src/services/event-outbox/event-outbox-dal.test.tsindex f528ae0796..cc4febbab1 100644--- a/backend/src/services/event-outbox/event-outbox-dal.test.ts+++ b/backend/src/services/event-outbox/event-outbox-dal.test.ts@@ -6,7 +6,7 @@ import { EventOutboxStatus } from "./event-outbox-types";// These assert query *shape*, in the style of alert-dal.test.ts. Only a real database can show that// two claimers never take the same row, so what a unit test protects is that the clauses buying that// guarantee are still there.-const buildDAL = () => {+const buildDAL = (opts?: { returning?: unknown[] }) => {const calls = {where: [] as unknown[][],whereIn: [] as unknown[][],@@ -87,7 +87,7 @@ const buildDAL = () => {calls.update.push(patch);return chain;},- returning: async () => [],+ returning: async () => opts?.returning ?? [],del: async () => 0,// Returns the chain so a subquery keeps building; awaiting it resolves through `then` below.select: () => chain,@@ -158,6 +158,19 @@ describe("event outbox dal", () => {expect(calls.orderBy[0]).toEqual(["id", "asc"]);});+ // RETURNING hands rows back in whatever order the UPDATE touched them, while handle() is promised id+ // order. Seen for real against Postgres: a 3-row claim came back 8, 7, 6.+ test("claimBatch returns the claimed rows in id order regardless of how RETURNING ordered them", async () => {+ const { dal } = buildDAL({ returning: [{ id: "8" }, { id: "6" }, { id: "10" }, { id: "7" }] });++ const claimed = await dal.claimBatch(+ { consumer: "alert", resourceType: "approval.workflow", resourceId: "policy-1" },+ 10+ );++ expect(claimed.map((row) => String(row.id))).toEqual(["6", "7", "8", "10"]);+ });+test("claimBatch flips the locked rows to processing in the same statement", async () => {const { dal, calls } = buildDAL();@@ -201,13 +214,31 @@ describe("event outbox dal", () => {test("findDueFlushKeys groups by the flush key and takes the most overdue first", async () => {const { dal, calls } = buildDAL();- await dal.findDueFlushKeys(200);+ await dal.findDueFlushKeys(200, ["alert"]);expect(calls.groupBy[0]).toEqual(["consumer", "resourceType", "resourceId"]);expect(calls.orderByRaw[0]).toBe('MIN("nextRetryAt") ASC');expect(calls.limit).toContain(200);});+ // Rows for a consumer this process can't drain would otherwise sort first on every tick and, once+ // there are more of them than the limit, hide every real key behind them.+ test("findDueFlushKeys only looks at consumers this process has registered", async () => {+ const { dal, calls } = buildDAL();++ await dal.findDueFlushKeys(200, ["alert", "audit"]);++ expect(calls.whereIn).toContainEqual(["consumer", ["alert", "audit"]]);+ });++ test("findDueFlushKeys issues no statement when nothing is registered", async () => {+ const { dal, calls } = buildDAL();++ await expect(dal.findDueFlushKeys(200, [])).resolves.toEqual([]);++ expect(calls.groupBy).toHaveLength(0);+ });+// Same clock the claim compares against, so app/DB skew can't shift the retry schedule.test("commitResults computes the next retry time in SQL, not in JS", async () => {const { dal, calls } = buildDAL();diff --git a/backend/src/services/event-outbox/event-outbox-dal.ts b/backend/src/services/event-outbox/event-outbox-dal.tsindex ab044e573c..a0ba0d1303 100644--- a/backend/src/services/event-outbox/event-outbox-dal.ts+++ b/backend/src/services/event-outbox/event-outbox-dal.ts@@ -4,7 +4,7 @@ import { TDbClient } from "@app/db";import { TableName } from "@app/db/schemas";import { DatabaseError } from "@app/lib/errors";-import { EventOutboxStatus, TEventOutboxRow, TOutboxFlushKey } from "./event-outbox-types";+import { computeBackoffMs, EventOutboxStatus, TEventOutboxRow, TOutboxFlushKey } from "./event-outbox-types";export type TEventOutboxDALFactory = ReturnType<typeof eventOutboxDALFactory>;@@ -40,10 +40,12 @@ export const eventOutboxDALFactory = (db: TDbClient) => {}};- const findDueFlushKeys = async (limit: number): Promise<TOutboxFlushKey[]> => {+ const findDueFlushKeys = async (limit: number, consumers: string[]): Promise<TOutboxFlushKey[]> => {+ if (consumers.length === 0) return [];try {const rows = await db.replicaNode()(TableName.EventOutbox)+ .whereIn("consumer", consumers).whereIn("status", [EventOutboxStatus.Pending, EventOutboxStatus.Retry]).andWhere("nextRetryAt", "<=", db.fn.now()).groupBy("consumer", "resourceType", "resourceId")@@ -69,8 +71,6 @@ export const eventOutboxDALFactory = (db: TDbClient) => {.where(key).whereIn("status", [EventOutboxStatus.Pending, EventOutboxStatus.Retry]).andWhere("nextRetryAt", "<=", db.fn.now())- // By id, never by a timestamp: a fresh row's nextRetryAt is its insert time, so sorting on- // that would let a later event overtake an earlier one that took a backoff..orderBy("id", "asc").limit(limit).forUpdate()@@ -79,7 +79,7 @@ export const eventOutboxDALFactory = (db: TDbClient) => {.update({ status: EventOutboxStatus.Processing, lockedAt: db.fn.now() }).returning("*");- return claimed as unknown as TEventOutboxRow[];+ return (claimed as unknown as TEventOutboxRow[]).sort((a, b) => (BigInt(a.id) < BigInt(b.id) ? -1 : 1));} catch (error) {throw new DatabaseError({ error, name: "EventOutbox: claimBatch" });}@@ -177,15 +177,22 @@ export const eventOutboxDALFactory = (db: TDbClient) => {if (staleRows.length === 0) return { retried: 0, failed: 0 };const exhausted = staleRows.filter((row) => row.attempts + 1 >= maxAttempts).map((row) => row.id);- const retriable = staleRows.filter((row) => row.attempts + 1 < maxAttempts).map((row) => row.id);+ const retriable = staleRows.filter((row) => row.attempts + 1 < maxAttempts);- if (retriable.length > 0) {+ const retriableByAttempt = new Map<number, string[]>();+ for (const row of retriable) {+ const ids = retriableByAttempt.get(row.attempts) ?? [];+ ids.push(row.id);+ retriableByAttempt.set(row.attempts, ids);+ }+ for (const [attempts, ids] of retriableByAttempt) {+ // eslint-disable-next-line no-await-in-loop -- one shared tx connection; writes are serialawait tx(TableName.EventOutbox)- .whereIn("id", retriable)+ .whereIn("id", ids).update({status: EventOutboxStatus.Retry,attempts: db.raw('"attempts" + 1'),- nextRetryAt: tx.fn.now(),+ nextRetryAt: db.raw(`NOW() + (? || ' milliseconds')::INTERVAL`, [computeBackoffMs(attempts + 1)]),lockedAt: null,lastError: "Worker did not report a result before the claim went stale"});diff --git a/backend/src/services/event-outbox/event-outbox-queue.test.ts b/backend/src/services/event-outbox/event-outbox-queue.test.tsindex abfee1a1ad..b4b3d21fe6 100644--- a/backend/src/services/event-outbox/event-outbox-queue.test.ts+++ b/backend/src/services/event-outbox/event-outbox-queue.test.ts@@ -16,6 +16,7 @@ const KEY: TOutboxFlushKey = { consumer: "alert", resourceType: "approval.workflconst buildQueue = (opts?: { keys?: TOutboxFlushKey[]; onDiscover?: () => Promise<void> }) => {const queued: { name: string; data: unknown; jobId?: string; attempts?: number }[] = [];+ const discoveredFor: string[][] = [];let discoverCalls = 0;const factory = eventOutboxQueueFactory({@@ -26,9 +27,11 @@ const buildQueue = (opts?: { keys?: TOutboxFlushKey[]; onDiscover?: () => Promisstart: () => {}} as never,cronJob: { register: () => {} } as never,+ eventOutboxRegistry: { names: () => ["alert", "audit"] },eventOutboxDAL: {- findDueFlushKeys: async () => {+ findDueFlushKeys: async (_limit: number, consumers: string[]) => {discoverCalls += 1;+ discoveredFor.push(consumers);if (opts?.onDiscover) await opts.onDiscover();return opts?.keys ?? [];},@@ -41,10 +44,18 @@ const buildQueue = (opts?: { keys?: TOutboxFlushKey[]; onDiscover?: () => Promis} as never});- return { factory, queued, getDiscoverCalls: () => discoverCalls };+ return { factory, queued, discoveredFor, getDiscoverCalls: () => discoverCalls };};describe("event outbox relay", () => {+ test("discovers work only for the consumers registered in this process", async () => {+ const { factory, discoveredFor } = buildQueue();++ await factory.runRelayTick();++ expect(discoveredFor).toEqual([["alert", "audit"]]);+ });+test("enqueues one flush job per discovered key", async () => {const { factory, queued } = buildQueue({ keys: [KEY, { ...KEY, resourceId: "policy-2" }] });diff --git a/backend/src/services/event-outbox/event-outbox-queue.ts b/backend/src/services/event-outbox/event-outbox-queue.tsindex 1a6ece4b38..834eea7748 100644--- a/backend/src/services/event-outbox/event-outbox-queue.ts+++ b/backend/src/services/event-outbox/event-outbox-queue.ts@@ -5,6 +5,7 @@ import { resolveCoreMeter } from "@app/lib/telemetry/metrics";import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";import { TEventOutboxDALFactory } from "./event-outbox-dal";+import { TEventOutboxRegistry } from "./event-outbox-registry";import { TEventOutboxServiceFactory } from "./event-outbox-service";import { OUTBOX_RELAY_INTERVAL_MS, RELAY_DISCOVERY_LIMIT, TOutboxFlushKey } from "./event-outbox-types";@@ -19,6 +20,7 @@ const RELAY_START_JITTER_MS = 5_000;export type TEventOutboxQueueFactoryDep = {queueService: TQueueServiceFactory;cronJob: TCronJobFactory;+ eventOutboxRegistry: Pick<TEventOutboxRegistry, "names">;eventOutboxDAL: Pick<TEventOutboxDALFactory, "findDueFlushKeys" | "findOldestPendingAgeSeconds">;eventOutboxService: Pick<TEventOutboxServiceFactory, "drain" | "sweepStaleClaims" | "pruneTerminalRows">;};@@ -26,6 +28,7 @@ export type TEventOutboxQueueFactoryDep = {export const eventOutboxQueueFactory = ({queueService,cronJob,+ eventOutboxRegistry,eventOutboxDAL,eventOutboxService}: TEventOutboxQueueFactoryDep) => {@@ -72,7 +75,7 @@ export const eventOutboxQueueFactory = ({});const runRelayTick = async () => {- const keys = await eventOutboxDAL.findDueFlushKeys(RELAY_DISCOVERY_LIMIT);+ const keys = await eventOutboxDAL.findDueFlushKeys(RELAY_DISCOVERY_LIMIT, eventOutboxRegistry.names());lastDiscoveryCount = keys.length;if (keys.length === RELAY_DISCOVERY_LIMIT) {diff --git a/backend/src/services/event-outbox/event-outbox-service.test.ts b/backend/src/services/event-outbox/event-outbox-service.test.tsindex 68cdc881c6..f53fc89b61 100644--- a/backend/src/services/event-outbox/event-outbox-service.test.ts+++ b/backend/src/services/event-outbox/event-outbox-service.test.ts@@ -2,7 +2,13 @@ import { z } from "zod";import { eventOutboxRegistryFactory } from "./event-outbox-registry";import { eventOutboxServiceFactory } from "./event-outbox-service";-import { EventOutboxStatus, IEventOutboxConsumer, TEventOutboxRow, TOutboxEvent } from "./event-outbox-types";+import {+ EventOutboxStatus,+ IEventOutboxConsumer,+ MAX_OUTBOX_ATTEMPTS,+ TEventOutboxRow,+ TOutboxEvent+} from "./event-outbox-types";vi.mock("@app/lib/logger", () => ({logger: { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} },@@ -232,7 +238,7 @@ describe("event outbox drain", () => {test("fails a row terminally once it has used up its attempts", async () => {const { service, commits } = buildDrain({- batches: [[makeRow({ attempts: 4 })]],+ batches: [[makeRow({ attempts: MAX_OUTBOX_ATTEMPTS - 1 })]],handle: async (rows) => rows.map((row) => ({ id: String(row.id), status: EventOutboxStatus.Retry })),consumerName: "alert"});@@ -323,4 +329,77 @@ describe("event outbox drain", () => {expect(result.unknownConsumer).toBe(true);expect(commits).toHaveLength(0);});++ // By this point the consumer has already sent. Letting a transient commit failure escape would leave+ // the rows to the stale sweeper and re-notify ten minutes later, so the commit itself is retried.+ test("retries a failed commit before giving up on it", async () => {+ vi.useFakeTimers();+ try {+ const registry = eventOutboxRegistryFactory();+ registry.register(+ makeConsumer({+ name: "alert",+ handle: async (rows) => rows.map((row) => ({ id: String(row.id), status: EventOutboxStatus.Delivered }))+ })+ );+ let commitCalls = 0;+ let batchServed = false;+ const service = eventOutboxServiceFactory({+ eventOutboxDAL: {+ claimBatch: async () => {+ if (batchServed) return [];+ batchServed = true;+ return [makeRow()];+ },+ extendClaims: async () => {},+ commitResults: async () => {+ commitCalls += 1;+ if (commitCalls === 1) throw new Error("connection reset");+ }+ } as never,+ eventOutboxRegistry: registry+ });++ const draining = service.drain(KEY);+ await vi.advanceTimersByTimeAsync(1_000);+ await expect(draining).resolves.toEqual({ handled: 1, unknownConsumer: false });+ expect(commitCalls).toBe(2);+ } finally {+ vi.useRealTimers();+ }+ });++ test("gives up on a commit that keeps failing so the sweeper can take over", async () => {+ vi.useFakeTimers();+ try {+ const registry = eventOutboxRegistryFactory();+ registry.register(+ makeConsumer({+ name: "alert",+ handle: async (rows) => rows.map((row) => ({ id: String(row.id), status: EventOutboxStatus.Delivered }))+ })+ );+ let commitCalls = 0;+ const service = eventOutboxServiceFactory({+ eventOutboxDAL: {+ claimBatch: async () => [makeRow()],+ extendClaims: async () => {},+ commitResults: async () => {+ commitCalls += 1;+ throw new Error("connection reset");+ }+ } as never,+ eventOutboxRegistry: registry+ });++ const draining = service.drain(KEY);+ // Attach the handler before advancing so the rejection has a listener when the retries run out.+ const outcome = expect(draining).rejects.toThrow("connection reset");+ await vi.advanceTimersByTimeAsync(5_000);+ await outcome;+ expect(commitCalls).toBe(3);+ } finally {+ vi.useRealTimers();+ }+ });});diff --git a/backend/src/services/event-outbox/event-outbox-service.ts b/backend/src/services/event-outbox/event-outbox-service.tsindex 0f3fc23929..2953de63f9 100644--- a/backend/src/services/event-outbox/event-outbox-service.ts+++ b/backend/src/services/event-outbox/event-outbox-service.ts@@ -7,6 +7,7 @@ import { recordEventOutboxExhaustedMetric, recordEventOutboxLagMetric } from "@aimport { TEventOutboxDALFactory, TOutboxInsertRow } from "./event-outbox-dal";import { TEventOutboxRegistry } from "./event-outbox-registry";import {+ computeBackoffMs,DELIVERED_RETENTION_MS,EventOutboxStatus,FAILED_RETENTION_MS,@@ -14,7 +15,6 @@ import {MAX_BATCHES_PER_FLUSH,MAX_OUTBOX_ATTEMPTS,MAX_OUTBOX_PAYLOAD_BYTES,- OUTBOX_BACKOFF_BASE_MS,OUTBOX_CLAIM_BATCH_SIZE,OutboxEventSchema,STALE_CLAIM_THRESHOLD_MS,@@ -31,12 +31,13 @@ const PRUNE_MAX_BATCHES = 20;// call a live claim stale.const CLAIM_HEARTBEAT_INTERVAL_MS = STALE_CLAIM_THRESHOLD_MS / 4;-const computeBackoffMs = (attemptsAfterIncrement: number): number => {- const exponent = Math.min(attemptsAfterIncrement - 1, 10);- const base = OUTBOX_BACKOFF_BASE_MS * 2 ** exponent;- const jitter = Math.floor(Math.random() * Math.min(base / 2, 30_000));- return base + jitter;-};+const COMMIT_ATTEMPTS = 3;+const COMMIT_RETRY_DELAY_MS = 250;++const sleep = (ms: number) =>+ new Promise<void>((resolve) => {+ setTimeout(resolve, ms);+ });const formatIssues = (issues: { path: (string | number)[]; message: string }[]) =>issues.map((issue) => `${issue.path.join(".")} ${issue.message}`).join(", ");@@ -127,6 +128,27 @@ export const eventOutboxServiceFactory = ({ eventOutboxDAL, eventOutboxRegistryawait eventOutboxDAL.insertEvents(rows, tx);};+ const $commitWithRetry = async (+ { consumer, resourceType, resourceId }: TOutboxFlushKey,+ input: Parameters<TEventOutboxDALFactory["commitResults"]>[0]+ ): Promise<void> => {+ for (let attempt = 1; ; attempt += 1) {+ try {+ // eslint-disable-next-line no-await-in-loop -- retrying the same statement is the point+ await eventOutboxDAL.commitResults(input);+ return;+ } catch (error) {+ if (attempt >= COMMIT_ATTEMPTS) throw error;+ logger.warn(+ error,+ `event-outbox: failed to commit results, retrying [attempt=${attempt}] [consumer=${consumer}] [resourceType=${resourceType}] [resourceId=${resourceId}]`+ );+ // eslint-disable-next-line no-await-in-loop -- see above+ await sleep(COMMIT_RETRY_DELAY_MS * attempt);+ }+ }+ };+const $applyResults = async (rows: TEventOutboxRow[], results: TOutboxRowResult[]): Promise<void> => {const byId = new Map(results.map((result) => [result.id, result]));const now = Date.now();@@ -176,7 +198,7 @@ export const eventOutboxServiceFactory = ({ eventOutboxDAL, eventOutboxRegistry});}- await eventOutboxDAL.commitResults({+ await $commitWithRetry(rows[0], {delivered: groupByOutcome(delivered, (item) => JSON.stringify(item.progress ?? null)),retriable: groupByOutcome(retriable, (item) =>JSON.stringify([item.nextRetryDelayMs, item.error ?? null, item.progress ?? null])diff --git a/backend/src/services/event-outbox/event-outbox-types.ts b/backend/src/services/event-outbox/event-outbox-types.tsindex c22d3b467a..fe5b46f26f 100644--- a/backend/src/services/event-outbox/event-outbox-types.ts+++ b/backend/src/services/event-outbox/event-outbox-types.ts@@ -22,10 +22,17 @@ export const MAX_BATCHES_PER_FLUSH = 10;export const OUTBOX_CLAIM_BATCH_SIZE = 100;-export const MAX_OUTBOX_ATTEMPTS = 5;+export const MAX_OUTBOX_ATTEMPTS = 8;export const OUTBOX_BACKOFF_BASE_MS = 30_000;+export const computeBackoffMs = (attemptsAfterIncrement: number): number => {+ const exponent = Math.min(attemptsAfterIncrement - 1, 10);+ const base = OUTBOX_BACKOFF_BASE_MS * 2 ** exponent;+ const jitter = Math.floor(Math.random() * Math.min(base / 2, 30_000));+ return base + jitter;+};+export const STALE_CLAIM_THRESHOLD_MS = 10 * 60_000;export const DELIVERED_RETENTION_MS = 24 * 60 * 60_000;