Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.
TypeScript acmeagentsai-agentscertificate-managementdevsecopsenvironment-variableskubernetesopen-sourcepampkipostgresprivate-caprivileged-access-managementsecret-managementsecret-managersecretssecrets-managementsecuritysecurity-toolsvault
lock claim
Thiago Araujo da Silvad86d255ecc29
diff --git a/backend/CLAUDE.md b/backend/CLAUDE.mdindex 8e145d6790..68125975bc 100644--- a/backend/CLAUDE.md+++ b/backend/CLAUDE.md@@ -682,9 +682,16 @@ await someDAL.transaction(async (tx) => {Backoff is exponential with jitter from 30s, and `MAX_OUTBOX_ATTEMPTS` puts the last attempt about anhour after the first, because a `failed` row is a notification nobody will receive. To replay failedrows by hand: `status = 'retry', attempts = 0, nextRetryAt = now()`.-- **A claim is a lease.** The sweeper hands back any `processing` row whose `lockedAt` is older than- `STALE_CLAIM_THRESHOLD_MS`, with the same backoff as a normal failure, and counts exhausted rows on the- same metric. `drain` refreshes `lockedAt` while `handle` runs so a slow batch isn't delivered twice.+- **A claim is a lease, and the lease is fenced.** The sweeper hands back any `processing` row whose+ `lockedAt` is older than `STALE_CLAIM_THRESHOLD_MS`, with the same backoff as a normal failure, and counts+ exhausted rows on the same metric. `drain` refreshes `lockedAt` while `handle` runs so a slow batch isn't+ delivered twice. Because `handle` has no time bound (unlike the audit log stream outbox, where every+ provider call has an HTTP timeout and a claim therefore can't outlive the threshold), that heartbeat can+ fail while the work carries on, so a claim can be recycled under a worker that is still alive. `claimBatch`+ stamps a `lockToken` and `extendClaims` / `commitResults` both require it, so the recycled worker's late+ result can't clear the new owner's lock or drop its outcome. `commitResults` returns how many rows it+ settled and `drain` logs a short settle: that count is the only signal that a batch went out twice, since+ the fence protects the bookkeeping but delivery stays at-least-once.- **Discovery only looks at consumers registered in this process.** Rows for any other name would sortfirst forever and, past the limit, hide every real key. They wait and show up on the oldest-pendinggauge instead.diff --git a/backend/e2e-test/event-outbox.spec.ts b/backend/e2e-test/event-outbox.spec.tsindex 721df70ebb..04283a1acc 100644--- a/backend/e2e-test/event-outbox.spec.ts+++ b/backend/e2e-test/event-outbox.spec.ts@@ -63,8 +63,8 @@ describe("event outbox (postgres)", () => {// Ids are bigints and arrive as strings, so order has to be checked numerically: "9" sorts after// "10" as text.- const idsA = a.map((row) => Number(row.id));- const idsB = b.map((row) => Number(row.id));+ const idsA = a.rows.map((row) => Number(row.id));+ const idsB = b.rows.map((row) => Number(row.id));expect(new Set([...idsA, ...idsB]).size).toBe(idsA.length + idsB.length);expect(idsA.length + idsB.length).toBe(6);expect([...idsA].sort((x, y) => x - y)).toEqual(idsA);@@ -72,7 +72,10 @@ describe("event outbox (postgres)", () => {const stored = await rowsFor();expect(stored.every((row) => row.status === EventOutboxStatus.Processing && row.lockedAt)).toBe(true);- expect(await dal.claimBatch(KEY, 10)).toEqual([]);+ // Each claim owns its rows outright, so the two claims can't share a token.+ expect(a.lockToken).not.toBe(b.lockToken);+ expect(new Set(stored.map((row) => row.lockToken))).toEqual(new Set([a.lockToken, b.lockToken]));+ expect((await dal.claimBatch(KEY, 10)).rows).toEqual([]);});test("claimBatch skips a row still inside its backoff window", async () => {@@ -82,24 +85,30 @@ describe("event outbox (postgres)", () => {.where("id", first.id).update({ status: EventOutboxStatus.Retry, nextRetryAt: new Date(Date.now() + 60 * 60_000) });- const claimed = await dal.claimBatch(KEY, 10);+ const { rows: claimed } = await dal.claimBatch(KEY, 10);expect(claimed.map((row) => String(row.id))).toEqual([String(second.id)]);});test("commitResults applies every outcome and computes the retry clock in the database", async () => {await insert([makeRow(), makeRow(), makeRow()]);- const [d, r, f] = await dal.claimBatch(KEY, 10);+ const {+ lockToken,+ rows: [d, r, f]+ } = await dal.claimBatch(KEY, 10);- await dal.commitResults({+ const settled = await dal.commitResults({+ lockToken,delivered: [{ ids: [String(d.id)], progress: { deliveredChannelIds: ["c-1"] } }],retriable: [{ ids: [String(r.id)], nextRetryDelayMs: 30 * 60_000, error: "slack 502" }],failed: [{ ids: [String(f.id)], error: "bad payload" }]});+ expect(settled).toBe(3);const [delivered, retry, failed] = await rowsFor();expect(delivered.status).toBe(EventOutboxStatus.Delivered);expect(delivered.lockedAt).toBeNull();+ expect(delivered.lockToken).toBeNull();expect(delivered.progress).toEqual({ deliveredChannelIds: ["c-1"] });expect(retry.status).toBe(EventOutboxStatus.Retry);@@ -132,7 +141,9 @@ describe("event outbox (postgres)", () => {test("recoverStaleClaims backs a live-looking claim off and fails an exhausted one", async () => {await insert([makeRow(), makeRow(), makeRow()]);- const [, stale, exhausted] = await dal.claimBatch(KEY, 10);+ const {+ rows: [, stale, exhausted]+ } = await dal.claimBatch(KEY, 10);const longAgo = new Date(Date.now() - 60 * 60_000);await testDb(TableName.EventOutbox).where("id", stale.id).update({ lockedAt: longAgo });await testDb(TableName.EventOutbox)@@ -151,6 +162,7 @@ describe("event outbox (postgres)", () => {expect(retried.status).toBe(EventOutboxStatus.Retry);expect(retried.attempts).toBe(1);expect(retried.lockedAt).toBeNull();+ expect(retried.lockToken).toBeNull();// Not handed straight back: a consumer that hangs every time must not hold a worker slot on repeat.expect(new Date(retried.nextRetryAt).getTime()).toBeGreaterThan(Date.now() + 20_000);expect(failed.status).toBe(EventOutboxStatus.Failed);@@ -159,33 +171,107 @@ describe("event outbox (postgres)", () => {test("commitResults ignores a late result for a claim the sweeper already handed back", async () => {await insert([makeRow()]);- const [row] = await dal.claimBatch(KEY, 10);+ const {+ lockToken,+ rows: [row]+ } = await dal.claimBatch(KEY, 10);await testDb(TableName.EventOutbox).where("id", row.id).update({ lockedAt: new Date(Date.now() - 60 * 60_000) });await dal.recoverStaleClaims({ thresholdMs: 10 * 60_000, maxAttempts: MAX_OUTBOX_ATTEMPTS, limit: 100 });- await dal.commitResults({ delivered: [{ ids: [String(row.id)] }], retriable: [], failed: [] });+ const settled = await dal.commitResults({+ lockToken,+ delivered: [{ ids: [String(row.id)] }],+ retriable: [],+ failed: []+ });+ expect(settled).toBe(0);const [stored] = await rowsFor();expect(stored.status).toBe(EventOutboxStatus.Retry);expect(stored.attempts).toBe(1);});- test("extendClaims refreshes only rows still processing", async () => {+ // The race the token exists for: the sweeper recycles a claim, a second worker picks the row up, and+ // only then does the first worker report. Status alone can't tell the two apart, because the second+ // worker's row is 'processing' too, so without the token the late result would mark it delivered and+ // the second worker's outcome would be dropped.+ test("commitResults leaves a row alone once another worker has reclaimed it", async () => {+ await insert([makeRow()]);+ const first = await dal.claimBatch(KEY, 10);+ await testDb(TableName.EventOutbox)+ .where("id", first.rows[0].id)+ .update({ lockedAt: new Date(Date.now() - 60 * 60_000) });+ await dal.recoverStaleClaims({ thresholdMs: 10 * 60_000, maxAttempts: MAX_OUTBOX_ATTEMPTS, limit: 100 });+ // Stand in for the backoff window elapsing.+ await testDb(TableName.EventOutbox).where("id", first.rows[0].id).update({ nextRetryAt: new Date() });+ const second = await dal.claimBatch(KEY, 10);+ expect(second.rows.map((row) => String(row.id))).toEqual([String(first.rows[0].id)]);++ const late = await dal.commitResults({+ lockToken: first.lockToken,+ delivered: [{ ids: [String(first.rows[0].id)] }],+ retriable: [],+ failed: []+ });++ expect(late).toBe(0);+ const [held] = await rowsFor();+ expect(held.status).toBe(EventOutboxStatus.Processing);+ expect(held.lockToken).toBe(second.lockToken);+ expect(held.lockedAt).not.toBeNull();++ // And the worker that actually holds the claim still settles it.+ expect(+ await dal.commitResults({+ lockToken: second.lockToken,+ delivered: [{ ids: [String(first.rows[0].id)] }],+ retriable: [],+ failed: []+ })+ ).toBe(1);+ const [settled] = await rowsFor();+ expect(settled.status).toBe(EventOutboxStatus.Delivered);+ expect(settled.lockToken).toBeNull();+ });++ test("extendClaims refreshes only rows this claim still holds", async () => {await insert([makeRow(), makeRow()]);- const [claimed, released] = await dal.claimBatch(KEY, 10);+ const {+ lockToken,+ rows: [claimed, released]+ } = await dal.claimBatch(KEY, 10);const longAgo = new Date(Date.now() - 60 * 60_000);await testDb(TableName.EventOutbox).whereIn("id", [claimed.id, released.id]).update({ lockedAt: longAgo });- await dal.commitResults({ delivered: [{ ids: [String(released.id)] }], retriable: [], failed: [] });+ await dal.commitResults({ lockToken, delivered: [{ ids: [String(released.id)] }], retriable: [], failed: [] });- await dal.extendClaims([String(claimed.id), String(released.id)]);+ await dal.extendClaims([String(claimed.id), String(released.id)], lockToken);const [a, b] = await rowsFor();expect(new Date(a.lockedAt as Date).getTime()).toBeGreaterThan(Date.now() - 10_000);expect(b.lockedAt).toBeNull();});+ // A heartbeat from the previous owner would otherwise keep the new owner's claim looking fresh, and+ // the sweeper would never recover it if that worker really is gone.+ test("extendClaims does not refresh a claim another worker now holds", async () => {+ await insert([makeRow()]);+ const first = await dal.claimBatch(KEY, 10);+ const longAgo = new Date(Date.now() - 60 * 60_000);+ await testDb(TableName.EventOutbox).where("id", first.rows[0].id).update({ lockedAt: longAgo });+ await dal.recoverStaleClaims({ thresholdMs: 10 * 60_000, maxAttempts: MAX_OUTBOX_ATTEMPTS, limit: 100 });+ await testDb(TableName.EventOutbox).where("id", first.rows[0].id).update({ nextRetryAt: new Date() });+ const second = await dal.claimBatch(KEY, 10);+ await testDb(TableName.EventOutbox).where("id", first.rows[0].id).update({ lockedAt: longAgo });++ await dal.extendClaims([String(first.rows[0].id)], first.lockToken);++ const [stored] = await rowsFor();+ expect(new Date(stored.lockedAt as Date).getTime()).toBe(longAgo.getTime());+ expect(stored.lockToken).toBe(second.lockToken);+ });+test("deleteTerminalOlderThan prunes only terminal rows past their retention, in bounded batches", async () => {const old = new Date(Date.now() - 48 * 60 * 60_000);await testDb(TableName.EventOutbox).insert([diff --git a/backend/src/db/migrations/20260908000000_event-outbox.ts b/backend/src/db/migrations/20260908000000_event-outbox.tsindex 2300ad85f5..eb468255ed 100644--- a/backend/src/db/migrations/20260908000000_event-outbox.ts+++ b/backend/src/db/migrations/20260908000000_event-outbox.ts@@ -6,7 +6,6 @@ import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";export async function up(knex: Knex): Promise<void> {if (!(await knex.schema.hasTable(TableName.EventOutbox))) {await knex.schema.createTable(TableName.EventOutbox, (t) => {- // Ordering within a resource is read off this column, so it has to be monotonic.t.bigIncrements("id").primary();t.string("consumer").notNullable();t.string("eventType").notNullable();@@ -14,7 +13,6 @@ export async function up(knex: Knex): Promise<void> {t.string("resourceId").notNullable();t.uuid("orgId").notNullable();t.string("projectId");- // Opaque here: each consumer validates it against its own payloadSchema at emit.t.jsonb("payload").notNullable();t.string("idempotencyKey");t.timestamp("occurredAt", { useTz: true }).notNullable().defaultTo(knex.fn.now());@@ -23,7 +21,7 @@ export async function up(knex: Knex): Promise<void> {t.integer("attempts").notNullable().defaultTo(0);t.timestamp("nextRetryAt", { useTz: true }).notNullable().defaultTo(knex.fn.now());t.timestamp("lockedAt", { useTz: true });- // Consumer-defined progress carried across attempts so a retry can skip what already landed.+ t.uuid("lockToken");t.jsonb("progress");t.text("lastError");t.timestamps(true, true, true);diff --git a/backend/src/db/schemas/event-outbox.ts b/backend/src/db/schemas/event-outbox.tsindex 3669811e8d..1df494dd0e 100644--- a/backend/src/db/schemas/event-outbox.ts+++ b/backend/src/db/schemas/event-outbox.ts@@ -22,6 +22,7 @@ export const EventOutboxSchema = z.object({attempts: z.number().default(0),nextRetryAt: z.date(),lockedAt: z.date().nullable().optional(),+ lockToken: z.string().uuid().nullable().optional(),progress: z.unknown().nullable().optional(),lastError: z.string().nullable().optional(),createdAt: z.date(),diff --git a/backend/src/services/event-outbox/event-outbox-dal.test.ts b/backend/src/services/event-outbox/event-outbox-dal.test.tsindex 126b7bec52..a02a625e5c 100644--- a/backend/src/services/event-outbox/event-outbox-dal.test.ts+++ b/backend/src/services/event-outbox/event-outbox-dal.test.ts@@ -6,7 +6,7 @@ import { EventOutboxStatus } from "./event-outbox-types";// These assert query *shape*, in the style of alert-dal.test.ts. Only a real database can show that// two claimers never take the same row, so what a unit test protects is that the clauses buying that// guarantee are still there.-const buildDAL = (opts?: { returning?: unknown[] }) => {+const buildDAL = (opts?: { returning?: unknown[]; updated?: number; selected?: unknown[] }) => {const calls = {where: [] as unknown[][],whereIn: [] as unknown[][],@@ -24,6 +24,10 @@ const buildDAL = (opts?: { returning?: unknown[] }) => {orderByRaw: [] as unknown[]};+ // knex resolves an UPDATE to the affected row count and a SELECT to rows, and commitResults reads+ // that count, so the mock has to answer each with the right shape.+ let lastOp: "select" | "update" = "select";+const chain: Record<string, unknown> = {};Object.assign(chain, {where: (...args: unknown[]) => {@@ -85,13 +89,18 @@ const buildDAL = (opts?: { returning?: unknown[] }) => {},update: (patch: Record<string, unknown>) => {calls.update.push(patch);+ lastOp = "update";return chain;},returning: async () => opts?.returning ?? [],del: async () => 0,// Returns the chain so a subquery keeps building; awaiting it resolves through `then` below.- select: () => chain,- then: (resolve: (v: unknown) => unknown) => resolve([])+ select: () => {+ lastOp = "select";+ return chain;+ },+ then: (resolve: (v: unknown) => unknown) =>+ resolve(lastOp === "update" ? (opts?.updated ?? 0) : (opts?.selected ?? []))});const queryBuilder = () => chain;@@ -163,12 +172,12 @@ describe("event outbox dal", () => {test("claimBatch returns the claimed rows in id order regardless of how RETURNING ordered them", async () => {const { dal } = buildDAL({ returning: [{ id: "8" }, { id: "6" }, { id: "10" }, { id: "7" }] });- const claimed = await dal.claimBatch(+ const { rows } = await dal.claimBatch({ consumer: "alert", resourceType: "approval.workflow", resourceId: "policy-1" },10);- expect(claimed.map((row) => String(row.id))).toEqual(["6", "7", "8", "10"]);+ expect(rows.map((row) => String(row.id))).toEqual(["6", "7", "8", "10"]);});test("claimBatch flips the locked rows to processing in the same statement", async () => {@@ -181,22 +190,37 @@ describe("event outbox dal", () => {expect(calls.update[0].lockedAt).toBe("NOW()");});+ // Every later statement about these rows carries the token, so it has to be the one stamped on the+ // rows and it has to differ per claim.+ test("claimBatch stamps a fresh lockToken and hands it back", async () => {+ const { dal, calls } = buildDAL();+ const key = { consumer: "alert", resourceType: "approval.workflow", resourceId: "policy-1" };++ const first = await dal.claimBatch(key, 100);+ const second = await dal.claimBatch(key, 100);++ expect(calls.update[0].lockToken).toBe(first.lockToken);+ expect(calls.update[1].lockToken).toBe(second.lockToken);+ expect(second.lockToken).not.toBe(first.lockToken);+ });+// A row the sweeper already handed back must not be pulled into 'processing' again by a late- // heartbeat.- test("extendClaims refreshes lockedAt only on rows still processing", async () => {+ // heartbeat, and a heartbeat must never refresh a claim another worker now holds.+ test("extendClaims refreshes lockedAt only on rows this claim still holds", async () => {const { dal, calls } = buildDAL();- await dal.extendClaims(["1", "2"]);+ await dal.extendClaims(["1", "2"], "token-1");expect(calls.whereIn[0]).toEqual(["id", ["1", "2"]]);expect(calls.where).toContainEqual(["status", EventOutboxStatus.Processing]);+ expect(calls.where).toContainEqual(["lockToken", "token-1"]);expect(calls.update[0]).toEqual({ lockedAt: "NOW()" });});test("extendClaims issues no statement for an empty claim", async () => {const { dal, calls } = buildDAL();- await dal.extendClaims([]);+ await dal.extendClaims([], "token-1");expect(calls.update).toHaveLength(0);});@@ -244,6 +268,7 @@ describe("event outbox dal", () => {const { dal, calls } = buildDAL();await dal.commitResults({+ lockToken: "token-1",delivered: [],retriable: [{ ids: ["1"], nextRetryDelayMs: 30_000 }],failed: []@@ -256,7 +281,7 @@ describe("event outbox dal", () => {test("commitResults issues no statement when there is nothing to commit", async () => {const { dal, calls } = buildDAL();- await dal.commitResults({ delivered: [], retriable: [], failed: [] });+ await dal.commitResults({ lockToken: "token-1", delivered: [], retriable: [], failed: [] });expect(calls.update).toHaveLength(0);});@@ -265,21 +290,24 @@ describe("event outbox dal", () => {const { dal, calls } = buildDAL();await dal.commitResults({+ lockToken: "token-1",delivered: [{ ids: ["1"] }],retriable: [],failed: [{ ids: ["2"], error: "gave up" }]});- expect(calls.update.every((patch) => patch.lockedAt === null)).toBe(true);+ expect(calls.update.every((patch) => patch.lockedAt === null && patch.lockToken === null)).toBe(true);expect(calls.update.map((patch) => patch.status)).toEqual([EventOutboxStatus.Delivered, EventOutboxStatus.Failed]);});// A claim the sweeper has already handed back may belong to another worker by the time the original- // one reports; its late result must not clobber the new owner's.- test("commitResults only touches rows still held by a processing claim", async () => {+ // one reports, and 'processing' alone doesn't tell the two claims apart: the new owner's rows are+ // 'processing' too. The token is what keeps the late result off them.+ test("commitResults only touches rows still held by this exact claim", async () => {const { dal, calls } = buildDAL();await dal.commitResults({+ lockToken: "token-1",delivered: [{ ids: ["1"] }],retriable: [{ ids: ["2"], nextRetryDelayMs: 1_000 }],failed: [{ ids: ["3"] }]@@ -289,6 +317,38 @@ describe("event outbox dal", () => {expect(calls.where.filter((args) => args[0] === "status" && args[1] === EventOutboxStatus.Processing)).toHaveLength(3);+ expect(calls.where.filter((args) => args[0] === "lockToken" && args[1] === "token-1")).toHaveLength(3);+ });++ // What the service reads to tell a settled claim from one that was taken away mid-delivery.+ test("commitResults reports how many rows it settled", async () => {+ const { dal } = buildDAL({ updated: 2 });++ await expect(+ dal.commitResults({+ lockToken: "token-1",+ delivered: [{ ids: ["1", "2"] }],+ retriable: [],+ failed: []+ })+ ).resolves.toBe(2);+ });++ // The rows go back to whoever claims them next, so leaving the old owner's token on them would let a+ // heartbeat that arrives even later keep refreshing a claim nobody holds.+ test("recoverStaleClaims clears the token along with the claim", async () => {+ const { dal, calls } = buildDAL({+ selected: [+ { id: "1", consumer: "alert", attempts: 0 },+ { id: "2", consumer: "alert", attempts: 4 }+ ]+ });++ await dal.recoverStaleClaims({ thresholdMs: 600_000, maxAttempts: 5, limit: 1_000 });++ expect(calls.update).toHaveLength(2);++ expect(calls.update.every((patch) => patch.lockedAt === null && patch.lockToken === null)).toBe(true);});test("recoverStaleClaims looks only at claims older than the threshold", async () => {diff --git a/backend/src/services/event-outbox/event-outbox-dal.ts b/backend/src/services/event-outbox/event-outbox-dal.tsindex c90c4b36e7..3b0cab94fa 100644--- a/backend/src/services/event-outbox/event-outbox-dal.ts+++ b/backend/src/services/event-outbox/event-outbox-dal.ts@@ -1,3 +1,5 @@+import crypto from "node:crypto";+import { Knex } from "knex";import { TDbClient } from "@app/db";@@ -8,6 +10,11 @@ import { computeBackoffMs, EventOutboxStatus, TOutboxFlushKey } from "./event-ouexport type TEventOutboxDALFactory = ReturnType<typeof eventOutboxDALFactory>;+export type TOutboxClaim = {+ lockToken: string;+ rows: TEventOutbox[];+};+export type TOutboxInsertRow = {consumer: string;eventType: string;@@ -59,8 +66,11 @@ export const eventOutboxDALFactory = (db: TDbClient) => {}};- // One statement on purpose: the row locks live only as long as the UPDATE.- const claimBatch = async (key: TOutboxFlushKey, limit: number): Promise<TEventOutbox[]> => {+ // One statement on purpose: the row locks live only as long as the UPDATE. The claim also stamps a+ // fresh lockToken, which is what lets extendClaims and commitResults tell this claim apart from a+ // later one over the same rows.+ const claimBatch = async (key: TOutboxFlushKey, limit: number): Promise<TOutboxClaim> => {+ const lockToken = crypto.randomUUID();try {const claimed = await db(TableName.EventOutbox).whereIn("id", (qb) => {@@ -75,48 +85,57 @@ export const eventOutboxDALFactory = (db: TDbClient) => {.forUpdate().skipLocked();})- .update({ status: EventOutboxStatus.Processing, lockedAt: db.fn.now() })+ .update({ status: EventOutboxStatus.Processing, lockedAt: db.fn.now(), lockToken }).returning("*");// RETURNING order is arbitrary, and handle() is promised id order.- return (claimed as unknown as TEventOutbox[]).sort((a, b) => (BigInt(a.id) < BigInt(b.id) ? -1 : 1));+ const rows = (claimed as unknown as TEventOutbox[]).sort((a, b) => (BigInt(a.id) < BigInt(b.id) ? -1 : 1));+ return { lockToken, rows };} catch (error) {throw new DatabaseError({ error, name: "EventOutbox: claimBatch" });}};// A batch that outlives the stale threshold has to keep lockedAt fresh or the sweeper hands its rows- // to another worker mid-delivery.- const extendClaims = async (ids: string[]): Promise<void> => {+ // to another worker mid-delivery. Scoped to the caller's own claim, so a heartbeat that comes back+ // after the sweeper recycled these rows can't refresh the new owner's lock.+ const extendClaims = async (ids: string[], lockToken: string): Promise<void> => {if (ids.length === 0) return;try {await db(TableName.EventOutbox).whereIn("id", ids).where("status", EventOutboxStatus.Processing)+ .where("lockToken", lockToken).update({ lockedAt: db.fn.now() });} catch (error) {throw new DatabaseError({ error, name: "EventOutbox: extendClaims" });}};+ // Returns how many rows were settled, which is short of what the caller claimed when the claim was+ // recycled mid-delivery and the rows now belong to another worker.const commitResults = async (input: {+ lockToken: string;delivered: { ids: string[]; progress?: Record<string, unknown> | null }[];retriable: { ids: string[]; nextRetryDelayMs: number; progress?: Record<string, unknown> | null; error?: string }[];failed: { ids: string[]; error?: string }[];- }): Promise<void> => {+ }): Promise<number> => {const total =input.delivered.reduce((n, g) => n + g.ids.length, 0) +input.retriable.reduce((n, g) => n + g.ids.length, 0) +input.failed.reduce((n, g) => n + g.ids.length, 0);- if (total === 0) return;+ if (total === 0) return 0;try {- await db.transaction(async (tx) => {- const settle = (ids: string[], values: Record<string, unknown>) =>- tx(TableName.EventOutbox)+ return await db.transaction(async (tx) => {+ let settled = 0;+ const settle = async (ids: string[], values: Record<string, unknown>) => {+ settled += await tx(TableName.EventOutbox).whereIn("id", ids).where("status", EventOutboxStatus.Processing)- .update({ lockedAt: null, ...values });+ .where("lockToken", input.lockToken)+ .update({ lockedAt: null, lockToken: null, ...values });+ };const progressOf = (group: { progress?: Record<string, unknown> | null }) =>group.progress !== undefined ? { progress: JSON.stringify(group.progress) } : {};@@ -145,6 +164,8 @@ export const eventOutboxDALFactory = (db: TDbClient) => {lastError: group.error ?? null});}++ return settled;});} catch (error) {throw new DatabaseError({ error, name: "EventOutbox: commitResults" });@@ -188,6 +209,7 @@ export const eventOutboxDALFactory = (db: TDbClient) => {attempts: db.raw('"attempts" + 1'),nextRetryAt: db.raw(`NOW() + (? || ' milliseconds')::INTERVAL`, [computeBackoffMs(attempts + 1)]),lockedAt: null,+ lockToken: null,lastError: "Worker did not report a result before the claim went stale"});}@@ -199,6 +221,7 @@ export const eventOutboxDALFactory = (db: TDbClient) => {status: EventOutboxStatus.Failed,attempts: db.raw('"attempts" + 1'),lockedAt: null,+ lockToken: null,lastError: "Worker did not report a result before the claim went stale"});}diff --git a/backend/src/services/event-outbox/event-outbox-service.test.ts b/backend/src/services/event-outbox/event-outbox-service.test.tsindex ebbbe7f825..6bf8cdc5b9 100644--- a/backend/src/services/event-outbox/event-outbox-service.test.ts+++ b/backend/src/services/event-outbox/event-outbox-service.test.ts@@ -12,8 +12,9 @@ import {TOutboxEvent} from "./event-outbox-types";+const { loggerWarn } = vi.hoisted(() => ({ loggerWarn: vi.fn<(...args: unknown[]) => void>() }));vi.mock("@app/lib/logger", () => ({- logger: { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} },+ logger: { info: () => {}, warn: loggerWarn, error: () => {}, debug: () => {} },initLogger: () => {}}));@@ -171,6 +172,7 @@ describe("event outbox drain", () => {batches: TEventOutbox[][];handle: IEventOutboxConsumer["handle"];consumerName?: string;+ settled?: (input: { delivered: { ids: string[] }[] }) => number;}) => {const registry = eventOutboxRegistryFactory();registry.register(makeConsumer({ name: opts.consumerName ?? "alert", handle: opts.handle }));@@ -180,27 +182,40 @@ describe("event outbox drain", () => {retriable: { ids: string[]; nextRetryDelayMs: number; error?: string }[];failed: { ids: string[]; error?: string }[];}[] = [];- const extended: string[][] = [];+ const extended: { ids: string[]; lockToken: string }[] = [];+ const claimTokens: string[] = [];let batchIdx = 0;const service = eventOutboxServiceFactory({eventOutboxDAL: {claimBatch: async () => {- const batch = opts.batches[batchIdx] ?? [];+ const rows = opts.batches[batchIdx] ?? [];batchIdx += 1;- return batch;+ const lockToken = `token-${batchIdx}`;+ claimTokens.push(lockToken);+ return { lockToken, rows };},- extendClaims: async (ids: string[]) => {- extended.push(ids);+ extendClaims: async (ids: string[], lockToken: string) => {+ extended.push({ ids, lockToken });},commitResults: async (input: never) => {commits.push(input);+ const counted = input as unknown as {+ delivered: { ids: string[] }[];+ retriable: { ids: string[] }[];+ failed: { ids: string[] }[];+ };+ if (opts.settled) return opts.settled(counted);+ return [...counted.delivered, ...counted.retriable, ...counted.failed].reduce(+ (n, group) => n + group.ids.length,+ 0+ );}} as never,eventOutboxRegistry: registry});- return { service, commits, extended };+ return { service, commits, extended, claimTokens };};const KEY = { consumer: "alert", resourceType: "approval.workflow", resourceId: "policy-1" };@@ -296,13 +311,54 @@ describe("event outbox drain", () => {expect(commits[0].delivered.map((group) => group.ids)).toEqual([["1"], ["2"]]);});+ // The token the rows were claimed under is what commitResults fences on, so a claim that reports+ // under a different one would write to whatever worker holds the rows now.+ test("commits under the token the batch was claimed with", async () => {+ const { service, commits, claimTokens } = buildDrain({+ batches: [[makeRow()]],+ handle: async (rows) => rows.map((row) => ({ id: String(row.id), status: EventOutboxStatus.Delivered }))+ });++ await service.drain(KEY);++ expect((commits[0] as unknown as { lockToken: string }).lockToken).toBe(claimTokens[0]);+ });++ // A short settle means the sweeper handed these rows to another worker while this one was still+ // delivering, so the batch went out twice. Nothing else in the system can see that happened.+ test("reports a claim that was taken away mid-delivery", async () => {+ loggerWarn.mockClear();+ const { service } = buildDrain({+ batches: [[makeRow({ id: 1 }), makeRow({ id: 2 })]],+ handle: async (rows) => rows.map((row) => ({ id: String(row.id), status: EventOutboxStatus.Delivered })),+ settled: () => 1+ });++ await service.drain(KEY);++ expect(loggerWarn).toHaveBeenCalledTimes(1);+ expect(loggerWarn.mock.calls[0][0]).toContain("settled 1 of 2 row(s)");+ });++ test("says nothing when every claimed row settles", async () => {+ loggerWarn.mockClear();+ const { service } = buildDrain({+ batches: [[makeRow()]],+ handle: async (rows) => rows.map((row) => ({ id: String(row.id), status: EventOutboxStatus.Delivered }))+ });++ await service.drain(KEY);++ expect(loggerWarn).not.toHaveBeenCalled();+ });+// A batch that legitimately outlives the sweeper's threshold (a slow webhook, many rows) has to// keep its claim fresh, or the sweeper hands its rows to a second worker mid-delivery.test("extends the claim while the consumer is still handling a batch", async () => {vi.useFakeTimers();try {let release: () => void = () => {};- const { service, extended } = buildDrain({+ const { service, extended, claimTokens } = buildDrain({batches: [[makeRow({ id: 1 }), makeRow({ id: 2 })]],handle: (rows) =>new Promise((resolve) => {@@ -314,7 +370,7 @@ describe("event outbox drain", () => {const draining = service.drain(KEY);await vi.advanceTimersByTimeAsync(10 * 60_000);expect(extended.length).toBeGreaterThanOrEqual(3);- expect(extended[0]).toEqual(["1", "2"]);+ expect(extended[0]).toEqual({ ids: ["1", "2"], lockToken: claimTokens[0] });release();await draining;@@ -357,14 +413,15 @@ describe("event outbox drain", () => {const service = eventOutboxServiceFactory({eventOutboxDAL: {claimBatch: async () => {- if (batchServed) return [];+ if (batchServed) return { lockToken: "token-1", rows: [] };batchServed = true;- return [makeRow()];+ return { lockToken: "token-1", rows: [makeRow()] };},extendClaims: async () => {},commitResults: async () => {commitCalls += 1;if (commitCalls === 1) throw new Error("connection reset");+ return 1;}} as never,eventOutboxRegistry: registry@@ -392,7 +449,7 @@ describe("event outbox drain", () => {let commitCalls = 0;const service = eventOutboxServiceFactory({eventOutboxDAL: {- claimBatch: async () => [makeRow()],+ claimBatch: async () => ({ lockToken: "token-1", rows: [makeRow()] }),extendClaims: async () => {},commitResults: async () => {commitCalls += 1;diff --git a/backend/src/services/event-outbox/event-outbox-service.ts b/backend/src/services/event-outbox/event-outbox-service.tsindex ed90bcc222..12d3058b02 100644--- a/backend/src/services/event-outbox/event-outbox-service.ts+++ b/backend/src/services/event-outbox/event-outbox-service.ts@@ -125,12 +125,11 @@ export const eventOutboxServiceFactory = ({ eventOutboxDAL, eventOutboxRegistryconst $commitWithRetry = async ({ consumer, resourceType, resourceId }: TOutboxFlushKey,input: Parameters<TEventOutboxDALFactory["commitResults"]>[0]- ): Promise<void> => {+ ): Promise<number> => {for (let attempt = 1; ; attempt += 1) {try {// eslint-disable-next-line no-await-in-loop -- retrying the same statement is the point- await eventOutboxDAL.commitResults(input);- return;+ return await eventOutboxDAL.commitResults(input);} catch (error) {if (attempt >= COMMIT_ATTEMPTS) throw error;logger.warn(@@ -143,7 +142,7 @@ export const eventOutboxServiceFactory = ({ eventOutboxDAL, eventOutboxRegistry}};- const $applyResults = async (rows: TEventOutbox[], results: TOutboxRowResult[]): Promise<void> => {+ const $applyResults = async (rows: TEventOutbox[], lockToken: string, results: TOutboxRowResult[]): Promise<void> => {const byId = new Map(results.map((result) => [result.id, result]));const now = Date.now();@@ -191,7 +190,8 @@ export const eventOutboxServiceFactory = ({ eventOutboxDAL, eventOutboxRegistry});}- await $commitWithRetry(rows[0], {+ const settled = await $commitWithRetry(rows[0], {+ lockToken,delivered: groupByOutcome(delivered, (item) => JSON.stringify(item.progress ?? null)),retriable: groupByOutcome(retriable, (item) =>JSON.stringify([item.nextRetryDelayMs, item.error ?? null, item.progress ?? null])@@ -199,6 +199,13 @@ export const eventOutboxServiceFactory = ({ eventOutboxDAL, eventOutboxRegistryfailed: groupByOutcome(failed, (item) => item.error ?? "")});+ if (settled < rows.length) {+ const { consumer, resourceType, resourceId } = rows[0];+ logger.warn(+ `event-outbox: settled ${settled} of ${rows.length} row(s); the rest were no longer held by this claim [consumer=${consumer}] [resourceType=${resourceType}] [resourceId=${resourceId}]`+ );+ }+if (failed.length > 0) {const { consumer, resourceType, resourceId } = rows[0];recordEventOutboxExhaustedMetric({ consumer, count: failed.length });@@ -213,11 +220,12 @@ export const eventOutboxServiceFactory = ({ eventOutboxDAL, eventOutboxRegistryconst $handleClaimed = async (consumer: IEventOutboxConsumer,key: TOutboxFlushKey,- claimed: TEventOutbox[]+ claimed: TEventOutbox[],+ lockToken: string): Promise<TOutboxRowResult[]> => {const ids = claimed.map((row) => String(row.id));const heartbeat = setInterval(() => {- eventOutboxDAL.extendClaims(ids).catch((error) => {+ eventOutboxDAL.extendClaims(ids, lockToken).catch((error) => {logger.warn(error,`event-outbox: failed to extend a claim [consumer=${key.consumer}] [resourceType=${key.resourceType}] [resourceId=${key.resourceId}]`@@ -253,13 +261,13 @@ export const eventOutboxServiceFactory = ({ eventOutboxDAL, eventOutboxRegistryfor (let batch = 0; batch < MAX_BATCHES_PER_FLUSH; batch += 1) {// eslint-disable-next-line no-await-in-loop -- batches are serial on purpose, to stay paced- const claimed = await eventOutboxDAL.claimBatch(key, OUTBOX_CLAIM_BATCH_SIZE);+ const { lockToken, rows: claimed } = await eventOutboxDAL.claimBatch(key, OUTBOX_CLAIM_BATCH_SIZE);if (claimed.length === 0) break;// eslint-disable-next-line no-await-in-loop -- see above- const results = await $handleClaimed(consumer, key, claimed);+ const results = await $handleClaimed(consumer, key, claimed, lockToken);// eslint-disable-next-line no-await-in-loop -- see above- await $applyResults(claimed, results);+ await $applyResults(claimed, lockToken, results);handled += claimed.length;}