Skip to content

Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.

lock claim

Thiago Araujo da Silvad86d255ecc29
diff --git a/backend/CLAUDE.md b/backend/CLAUDE.md
index 8e145d6790..68125975bc 100644
--- a/backend/CLAUDE.md
+++ b/backend/CLAUDE.md
@@ -682,9 +682,16 @@ await someDAL.transaction(async (tx) => {
Backoff is exponential with jitter from 30s, and `MAX_OUTBOX_ATTEMPTS` puts the last attempt about an
hour after the first, because a `failed` row is a notification nobody will receive. To replay failed
rows by hand: `status = 'retry', attempts = 0, nextRetryAt = now()`.
-- **A claim is a lease.** The sweeper hands back any `processing` row whose `lockedAt` is older than
- `STALE_CLAIM_THRESHOLD_MS`, with the same backoff as a normal failure, and counts exhausted rows on the
- same metric. `drain` refreshes `lockedAt` while `handle` runs so a slow batch isn't delivered twice.
+- **A claim is a lease, and the lease is fenced.** The sweeper hands back any `processing` row whose
+ `lockedAt` is older than `STALE_CLAIM_THRESHOLD_MS`, with the same backoff as a normal failure, and counts
+ exhausted rows on the same metric. `drain` refreshes `lockedAt` while `handle` runs so a slow batch isn't
+ delivered twice. Because `handle` has no time bound (unlike the audit log stream outbox, where every
+ provider call has an HTTP timeout and a claim therefore can't outlive the threshold), that heartbeat can
+ fail while the work carries on, so a claim can be recycled under a worker that is still alive. `claimBatch`
+ stamps a `lockToken` and `extendClaims` / `commitResults` both require it, so the recycled worker's late
+ result can't clear the new owner's lock or drop its outcome. `commitResults` returns how many rows it
+ settled and `drain` logs a short settle: that count is the only signal that a batch went out twice, since
+ the fence protects the bookkeeping but delivery stays at-least-once.
- **Discovery only looks at consumers registered in this process.** Rows for any other name would sort
first forever and, past the limit, hide every real key. They wait and show up on the oldest-pending
gauge instead.
diff --git a/backend/e2e-test/event-outbox.spec.ts b/backend/e2e-test/event-outbox.spec.ts
index 721df70ebb..04283a1acc 100644
--- a/backend/e2e-test/event-outbox.spec.ts
+++ b/backend/e2e-test/event-outbox.spec.ts
@@ -63,8 +63,8 @@ describe("event outbox (postgres)", () => {
// Ids are bigints and arrive as strings, so order has to be checked numerically: "9" sorts after
// "10" as text.
- const idsA = a.map((row) => Number(row.id));
- const idsB = b.map((row) => Number(row.id));
+ const idsA = a.rows.map((row) => Number(row.id));
+ const idsB = b.rows.map((row) => Number(row.id));
expect(new Set([...idsA, ...idsB]).size).toBe(idsA.length + idsB.length);
expect(idsA.length + idsB.length).toBe(6);
expect([...idsA].sort((x, y) => x - y)).toEqual(idsA);
@@ -72,7 +72,10 @@ describe("event outbox (postgres)", () => {
const stored = await rowsFor();
expect(stored.every((row) => row.status === EventOutboxStatus.Processing && row.lockedAt)).toBe(true);
- expect(await dal.claimBatch(KEY, 10)).toEqual([]);
+ // Each claim owns its rows outright, so the two claims can't share a token.
+ expect(a.lockToken).not.toBe(b.lockToken);
+ expect(new Set(stored.map((row) => row.lockToken))).toEqual(new Set([a.lockToken, b.lockToken]));
+ expect((await dal.claimBatch(KEY, 10)).rows).toEqual([]);
});
test("claimBatch skips a row still inside its backoff window", async () => {
@@ -82,24 +85,30 @@ describe("event outbox (postgres)", () => {
.where("id", first.id)
.update({ status: EventOutboxStatus.Retry, nextRetryAt: new Date(Date.now() + 60 * 60_000) });
- const claimed = await dal.claimBatch(KEY, 10);
+ const { rows: claimed } = await dal.claimBatch(KEY, 10);
expect(claimed.map((row) => String(row.id))).toEqual([String(second.id)]);
});
test("commitResults applies every outcome and computes the retry clock in the database", async () => {
await insert([makeRow(), makeRow(), makeRow()]);
- const [d, r, f] = await dal.claimBatch(KEY, 10);
+ const {
+ lockToken,
+ rows: [d, r, f]
+ } = await dal.claimBatch(KEY, 10);
- await dal.commitResults({
+ const settled = await dal.commitResults({
+ lockToken,
delivered: [{ ids: [String(d.id)], progress: { deliveredChannelIds: ["c-1"] } }],
retriable: [{ ids: [String(r.id)], nextRetryDelayMs: 30 * 60_000, error: "slack 502" }],
failed: [{ ids: [String(f.id)], error: "bad payload" }]
});
+ expect(settled).toBe(3);
const [delivered, retry, failed] = await rowsFor();
expect(delivered.status).toBe(EventOutboxStatus.Delivered);
expect(delivered.lockedAt).toBeNull();
+ expect(delivered.lockToken).toBeNull();
expect(delivered.progress).toEqual({ deliveredChannelIds: ["c-1"] });
expect(retry.status).toBe(EventOutboxStatus.Retry);
@@ -132,7 +141,9 @@ describe("event outbox (postgres)", () => {
test("recoverStaleClaims backs a live-looking claim off and fails an exhausted one", async () => {
await insert([makeRow(), makeRow(), makeRow()]);
- const [, stale, exhausted] = await dal.claimBatch(KEY, 10);
+ const {
+ rows: [, stale, exhausted]
+ } = await dal.claimBatch(KEY, 10);
const longAgo = new Date(Date.now() - 60 * 60_000);
await testDb(TableName.EventOutbox).where("id", stale.id).update({ lockedAt: longAgo });
await testDb(TableName.EventOutbox)
@@ -151,6 +162,7 @@ describe("event outbox (postgres)", () => {
expect(retried.status).toBe(EventOutboxStatus.Retry);
expect(retried.attempts).toBe(1);
expect(retried.lockedAt).toBeNull();
+ expect(retried.lockToken).toBeNull();
// Not handed straight back: a consumer that hangs every time must not hold a worker slot on repeat.
expect(new Date(retried.nextRetryAt).getTime()).toBeGreaterThan(Date.now() + 20_000);
expect(failed.status).toBe(EventOutboxStatus.Failed);
@@ -159,33 +171,107 @@ describe("event outbox (postgres)", () => {
test("commitResults ignores a late result for a claim the sweeper already handed back", async () => {
await insert([makeRow()]);
- const [row] = await dal.claimBatch(KEY, 10);
+ const {
+ lockToken,
+ rows: [row]
+ } = await dal.claimBatch(KEY, 10);
await testDb(TableName.EventOutbox)
.where("id", row.id)
.update({ lockedAt: new Date(Date.now() - 60 * 60_000) });
await dal.recoverStaleClaims({ thresholdMs: 10 * 60_000, maxAttempts: MAX_OUTBOX_ATTEMPTS, limit: 100 });
- await dal.commitResults({ delivered: [{ ids: [String(row.id)] }], retriable: [], failed: [] });
+ const settled = await dal.commitResults({
+ lockToken,
+ delivered: [{ ids: [String(row.id)] }],
+ retriable: [],
+ failed: []
+ });
+ expect(settled).toBe(0);
const [stored] = await rowsFor();
expect(stored.status).toBe(EventOutboxStatus.Retry);
expect(stored.attempts).toBe(1);
});
- test("extendClaims refreshes only rows still processing", async () => {
+ // The race the token exists for: the sweeper recycles a claim, a second worker picks the row up, and
+ // only then does the first worker report. Status alone can't tell the two apart, because the second
+ // worker's row is 'processing' too, so without the token the late result would mark it delivered and
+ // the second worker's outcome would be dropped.
+ test("commitResults leaves a row alone once another worker has reclaimed it", async () => {
+ await insert([makeRow()]);
+ const first = await dal.claimBatch(KEY, 10);
+ await testDb(TableName.EventOutbox)
+ .where("id", first.rows[0].id)
+ .update({ lockedAt: new Date(Date.now() - 60 * 60_000) });
+ await dal.recoverStaleClaims({ thresholdMs: 10 * 60_000, maxAttempts: MAX_OUTBOX_ATTEMPTS, limit: 100 });
+ // Stand in for the backoff window elapsing.
+ await testDb(TableName.EventOutbox).where("id", first.rows[0].id).update({ nextRetryAt: new Date() });
+ const second = await dal.claimBatch(KEY, 10);
+ expect(second.rows.map((row) => String(row.id))).toEqual([String(first.rows[0].id)]);
+
+ const late = await dal.commitResults({
+ lockToken: first.lockToken,
+ delivered: [{ ids: [String(first.rows[0].id)] }],
+ retriable: [],
+ failed: []
+ });
+
+ expect(late).toBe(0);
+ const [held] = await rowsFor();
+ expect(held.status).toBe(EventOutboxStatus.Processing);
+ expect(held.lockToken).toBe(second.lockToken);
+ expect(held.lockedAt).not.toBeNull();
+
+ // And the worker that actually holds the claim still settles it.
+ expect(
+ await dal.commitResults({
+ lockToken: second.lockToken,
+ delivered: [{ ids: [String(first.rows[0].id)] }],
+ retriable: [],
+ failed: []
+ })
+ ).toBe(1);
+ const [settled] = await rowsFor();
+ expect(settled.status).toBe(EventOutboxStatus.Delivered);
+ expect(settled.lockToken).toBeNull();
+ });
+
+ test("extendClaims refreshes only rows this claim still holds", async () => {
await insert([makeRow(), makeRow()]);
- const [claimed, released] = await dal.claimBatch(KEY, 10);
+ const {
+ lockToken,
+ rows: [claimed, released]
+ } = await dal.claimBatch(KEY, 10);
const longAgo = new Date(Date.now() - 60 * 60_000);
await testDb(TableName.EventOutbox).whereIn("id", [claimed.id, released.id]).update({ lockedAt: longAgo });
- await dal.commitResults({ delivered: [{ ids: [String(released.id)] }], retriable: [], failed: [] });
+ await dal.commitResults({ lockToken, delivered: [{ ids: [String(released.id)] }], retriable: [], failed: [] });
- await dal.extendClaims([String(claimed.id), String(released.id)]);
+ await dal.extendClaims([String(claimed.id), String(released.id)], lockToken);
const [a, b] = await rowsFor();
expect(new Date(a.lockedAt as Date).getTime()).toBeGreaterThan(Date.now() - 10_000);
expect(b.lockedAt).toBeNull();
});
+ // A heartbeat from the previous owner would otherwise keep the new owner's claim looking fresh, and
+ // the sweeper would never recover it if that worker really is gone.
+ test("extendClaims does not refresh a claim another worker now holds", async () => {
+ await insert([makeRow()]);
+ const first = await dal.claimBatch(KEY, 10);
+ const longAgo = new Date(Date.now() - 60 * 60_000);
+ await testDb(TableName.EventOutbox).where("id", first.rows[0].id).update({ lockedAt: longAgo });
+ await dal.recoverStaleClaims({ thresholdMs: 10 * 60_000, maxAttempts: MAX_OUTBOX_ATTEMPTS, limit: 100 });
+ await testDb(TableName.EventOutbox).where("id", first.rows[0].id).update({ nextRetryAt: new Date() });
+ const second = await dal.claimBatch(KEY, 10);
+ await testDb(TableName.EventOutbox).where("id", first.rows[0].id).update({ lockedAt: longAgo });
+
+ await dal.extendClaims([String(first.rows[0].id)], first.lockToken);
+
+ const [stored] = await rowsFor();
+ expect(new Date(stored.lockedAt as Date).getTime()).toBe(longAgo.getTime());
+ expect(stored.lockToken).toBe(second.lockToken);
+ });
+
test("deleteTerminalOlderThan prunes only terminal rows past their retention, in bounded batches", async () => {
const old = new Date(Date.now() - 48 * 60 * 60_000);
await testDb(TableName.EventOutbox).insert([
diff --git a/backend/src/db/migrations/20260908000000_event-outbox.ts b/backend/src/db/migrations/20260908000000_event-outbox.ts
index 2300ad85f5..eb468255ed 100644
--- a/backend/src/db/migrations/20260908000000_event-outbox.ts
+++ b/backend/src/db/migrations/20260908000000_event-outbox.ts
@@ -6,7 +6,6 @@ import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";
export async function up(knex: Knex): Promise<void> {
if (!(await knex.schema.hasTable(TableName.EventOutbox))) {
await knex.schema.createTable(TableName.EventOutbox, (t) => {
- // Ordering within a resource is read off this column, so it has to be monotonic.
t.bigIncrements("id").primary();
t.string("consumer").notNullable();
t.string("eventType").notNullable();
@@ -14,7 +13,6 @@ export async function up(knex: Knex): Promise<void> {
t.string("resourceId").notNullable();
t.uuid("orgId").notNullable();
t.string("projectId");
- // Opaque here: each consumer validates it against its own payloadSchema at emit.
t.jsonb("payload").notNullable();
t.string("idempotencyKey");
t.timestamp("occurredAt", { useTz: true }).notNullable().defaultTo(knex.fn.now());
@@ -23,7 +21,7 @@ export async function up(knex: Knex): Promise<void> {
t.integer("attempts").notNullable().defaultTo(0);
t.timestamp("nextRetryAt", { useTz: true }).notNullable().defaultTo(knex.fn.now());
t.timestamp("lockedAt", { useTz: true });
- // Consumer-defined progress carried across attempts so a retry can skip what already landed.
+ t.uuid("lockToken");
t.jsonb("progress");
t.text("lastError");
t.timestamps(true, true, true);
diff --git a/backend/src/db/schemas/event-outbox.ts b/backend/src/db/schemas/event-outbox.ts
index 3669811e8d..1df494dd0e 100644
--- a/backend/src/db/schemas/event-outbox.ts
+++ b/backend/src/db/schemas/event-outbox.ts
@@ -22,6 +22,7 @@ export const EventOutboxSchema = z.object({
attempts: z.number().default(0),
nextRetryAt: z.date(),
lockedAt: z.date().nullable().optional(),
+ lockToken: z.string().uuid().nullable().optional(),
progress: z.unknown().nullable().optional(),
lastError: z.string().nullable().optional(),
createdAt: z.date(),
diff --git a/backend/src/services/event-outbox/event-outbox-dal.test.ts b/backend/src/services/event-outbox/event-outbox-dal.test.ts
index 126b7bec52..a02a625e5c 100644
--- a/backend/src/services/event-outbox/event-outbox-dal.test.ts
+++ b/backend/src/services/event-outbox/event-outbox-dal.test.ts
@@ -6,7 +6,7 @@ import { EventOutboxStatus } from "./event-outbox-types";
// These assert query *shape*, in the style of alert-dal.test.ts. Only a real database can show that
// two claimers never take the same row, so what a unit test protects is that the clauses buying that
// guarantee are still there.
-const buildDAL = (opts?: { returning?: unknown[] }) => {
+const buildDAL = (opts?: { returning?: unknown[]; updated?: number; selected?: unknown[] }) => {
const calls = {
where: [] as unknown[][],
whereIn: [] as unknown[][],
@@ -24,6 +24,10 @@ const buildDAL = (opts?: { returning?: unknown[] }) => {
orderByRaw: [] as unknown[]
};
+ // knex resolves an UPDATE to the affected row count and a SELECT to rows, and commitResults reads
+ // that count, so the mock has to answer each with the right shape.
+ let lastOp: "select" | "update" = "select";
+
const chain: Record<string, unknown> = {};
Object.assign(chain, {
where: (...args: unknown[]) => {
@@ -85,13 +89,18 @@ const buildDAL = (opts?: { returning?: unknown[] }) => {
},
update: (patch: Record<string, unknown>) => {
calls.update.push(patch);
+ lastOp = "update";
return chain;
},
returning: async () => opts?.returning ?? [],
del: async () => 0,
// Returns the chain so a subquery keeps building; awaiting it resolves through `then` below.
- select: () => chain,
- then: (resolve: (v: unknown) => unknown) => resolve([])
+ select: () => {
+ lastOp = "select";
+ return chain;
+ },
+ then: (resolve: (v: unknown) => unknown) =>
+ resolve(lastOp === "update" ? (opts?.updated ?? 0) : (opts?.selected ?? []))
});
const queryBuilder = () => chain;
@@ -163,12 +172,12 @@ describe("event outbox dal", () => {
test("claimBatch returns the claimed rows in id order regardless of how RETURNING ordered them", async () => {
const { dal } = buildDAL({ returning: [{ id: "8" }, { id: "6" }, { id: "10" }, { id: "7" }] });
- const claimed = await dal.claimBatch(
+ const { rows } = await dal.claimBatch(
{ consumer: "alert", resourceType: "approval.workflow", resourceId: "policy-1" },
10
);
- expect(claimed.map((row) => String(row.id))).toEqual(["6", "7", "8", "10"]);
+ expect(rows.map((row) => String(row.id))).toEqual(["6", "7", "8", "10"]);
});
test("claimBatch flips the locked rows to processing in the same statement", async () => {
@@ -181,22 +190,37 @@ describe("event outbox dal", () => {
expect(calls.update[0].lockedAt).toBe("NOW()");
});
+ // Every later statement about these rows carries the token, so it has to be the one stamped on the
+ // rows and it has to differ per claim.
+ test("claimBatch stamps a fresh lockToken and hands it back", async () => {
+ const { dal, calls } = buildDAL();
+ const key = { consumer: "alert", resourceType: "approval.workflow", resourceId: "policy-1" };
+
+ const first = await dal.claimBatch(key, 100);
+ const second = await dal.claimBatch(key, 100);
+
+ expect(calls.update[0].lockToken).toBe(first.lockToken);
+ expect(calls.update[1].lockToken).toBe(second.lockToken);
+ expect(second.lockToken).not.toBe(first.lockToken);
+ });
+
// A row the sweeper already handed back must not be pulled into 'processing' again by a late
- // heartbeat.
- test("extendClaims refreshes lockedAt only on rows still processing", async () => {
+ // heartbeat, and a heartbeat must never refresh a claim another worker now holds.
+ test("extendClaims refreshes lockedAt only on rows this claim still holds", async () => {
const { dal, calls } = buildDAL();
- await dal.extendClaims(["1", "2"]);
+ await dal.extendClaims(["1", "2"], "token-1");
expect(calls.whereIn[0]).toEqual(["id", ["1", "2"]]);
expect(calls.where).toContainEqual(["status", EventOutboxStatus.Processing]);
+ expect(calls.where).toContainEqual(["lockToken", "token-1"]);
expect(calls.update[0]).toEqual({ lockedAt: "NOW()" });
});
test("extendClaims issues no statement for an empty claim", async () => {
const { dal, calls } = buildDAL();
- await dal.extendClaims([]);
+ await dal.extendClaims([], "token-1");
expect(calls.update).toHaveLength(0);
});
@@ -244,6 +268,7 @@ describe("event outbox dal", () => {
const { dal, calls } = buildDAL();
await dal.commitResults({
+ lockToken: "token-1",
delivered: [],
retriable: [{ ids: ["1"], nextRetryDelayMs: 30_000 }],
failed: []
@@ -256,7 +281,7 @@ describe("event outbox dal", () => {
test("commitResults issues no statement when there is nothing to commit", async () => {
const { dal, calls } = buildDAL();
- await dal.commitResults({ delivered: [], retriable: [], failed: [] });
+ await dal.commitResults({ lockToken: "token-1", delivered: [], retriable: [], failed: [] });
expect(calls.update).toHaveLength(0);
});
@@ -265,21 +290,24 @@ describe("event outbox dal", () => {
const { dal, calls } = buildDAL();
await dal.commitResults({
+ lockToken: "token-1",
delivered: [{ ids: ["1"] }],
retriable: [],
failed: [{ ids: ["2"], error: "gave up" }]
});
- expect(calls.update.every((patch) => patch.lockedAt === null)).toBe(true);
+ expect(calls.update.every((patch) => patch.lockedAt === null && patch.lockToken === null)).toBe(true);
expect(calls.update.map((patch) => patch.status)).toEqual([EventOutboxStatus.Delivered, EventOutboxStatus.Failed]);
});
// A claim the sweeper has already handed back may belong to another worker by the time the original
- // one reports; its late result must not clobber the new owner's.
- test("commitResults only touches rows still held by a processing claim", async () => {
+ // one reports, and 'processing' alone doesn't tell the two claims apart: the new owner's rows are
+ // 'processing' too. The token is what keeps the late result off them.
+ test("commitResults only touches rows still held by this exact claim", async () => {
const { dal, calls } = buildDAL();
await dal.commitResults({
+ lockToken: "token-1",
delivered: [{ ids: ["1"] }],
retriable: [{ ids: ["2"], nextRetryDelayMs: 1_000 }],
failed: [{ ids: ["3"] }]
@@ -289,6 +317,38 @@ describe("event outbox dal", () => {
expect(calls.where.filter((args) => args[0] === "status" && args[1] === EventOutboxStatus.Processing)).toHaveLength(
3
);
+ expect(calls.where.filter((args) => args[0] === "lockToken" && args[1] === "token-1")).toHaveLength(3);
+ });
+
+ // What the service reads to tell a settled claim from one that was taken away mid-delivery.
+ test("commitResults reports how many rows it settled", async () => {
+ const { dal } = buildDAL({ updated: 2 });
+
+ await expect(
+ dal.commitResults({
+ lockToken: "token-1",
+ delivered: [{ ids: ["1", "2"] }],
+ retriable: [],
+ failed: []
+ })
+ ).resolves.toBe(2);
+ });
+
+ // The rows go back to whoever claims them next, so leaving the old owner's token on them would let a
+ // heartbeat that arrives even later keep refreshing a claim nobody holds.
+ test("recoverStaleClaims clears the token along with the claim", async () => {
+ const { dal, calls } = buildDAL({
+ selected: [
+ { id: "1", consumer: "alert", attempts: 0 },
+ { id: "2", consumer: "alert", attempts: 4 }
+ ]
+ });
+
+ await dal.recoverStaleClaims({ thresholdMs: 600_000, maxAttempts: 5, limit: 1_000 });
+
+ expect(calls.update).toHaveLength(2);
+
+ expect(calls.update.every((patch) => patch.lockedAt === null && patch.lockToken === null)).toBe(true);
});
test("recoverStaleClaims looks only at claims older than the threshold", async () => {
diff --git a/backend/src/services/event-outbox/event-outbox-dal.ts b/backend/src/services/event-outbox/event-outbox-dal.ts
index c90c4b36e7..3b0cab94fa 100644
--- a/backend/src/services/event-outbox/event-outbox-dal.ts
+++ b/backend/src/services/event-outbox/event-outbox-dal.ts
@@ -1,3 +1,5 @@
+import crypto from "node:crypto";
+
import { Knex } from "knex";
import { TDbClient } from "@app/db";
@@ -8,6 +10,11 @@ import { computeBackoffMs, EventOutboxStatus, TOutboxFlushKey } from "./event-ou
export type TEventOutboxDALFactory = ReturnType<typeof eventOutboxDALFactory>;
+export type TOutboxClaim = {
+ lockToken: string;
+ rows: TEventOutbox[];
+};
+
export type TOutboxInsertRow = {
consumer: string;
eventType: string;
@@ -59,8 +66,11 @@ export const eventOutboxDALFactory = (db: TDbClient) => {
}
};
- // One statement on purpose: the row locks live only as long as the UPDATE.
- const claimBatch = async (key: TOutboxFlushKey, limit: number): Promise<TEventOutbox[]> => {
+ // One statement on purpose: the row locks live only as long as the UPDATE. The claim also stamps a
+ // fresh lockToken, which is what lets extendClaims and commitResults tell this claim apart from a
+ // later one over the same rows.
+ const claimBatch = async (key: TOutboxFlushKey, limit: number): Promise<TOutboxClaim> => {
+ const lockToken = crypto.randomUUID();
try {
const claimed = await db(TableName.EventOutbox)
.whereIn("id", (qb) => {
@@ -75,48 +85,57 @@ export const eventOutboxDALFactory = (db: TDbClient) => {
.forUpdate()
.skipLocked();
})
- .update({ status: EventOutboxStatus.Processing, lockedAt: db.fn.now() })
+ .update({ status: EventOutboxStatus.Processing, lockedAt: db.fn.now(), lockToken })
.returning("*");
// RETURNING order is arbitrary, and handle() is promised id order.
- return (claimed as unknown as TEventOutbox[]).sort((a, b) => (BigInt(a.id) < BigInt(b.id) ? -1 : 1));
+ const rows = (claimed as unknown as TEventOutbox[]).sort((a, b) => (BigInt(a.id) < BigInt(b.id) ? -1 : 1));
+ return { lockToken, rows };
} catch (error) {
throw new DatabaseError({ error, name: "EventOutbox: claimBatch" });
}
};
// A batch that outlives the stale threshold has to keep lockedAt fresh or the sweeper hands its rows
- // to another worker mid-delivery.
- const extendClaims = async (ids: string[]): Promise<void> => {
+ // to another worker mid-delivery. Scoped to the caller's own claim, so a heartbeat that comes back
+ // after the sweeper recycled these rows can't refresh the new owner's lock.
+ const extendClaims = async (ids: string[], lockToken: string): Promise<void> => {
if (ids.length === 0) return;
try {
await db(TableName.EventOutbox)
.whereIn("id", ids)
.where("status", EventOutboxStatus.Processing)
+ .where("lockToken", lockToken)
.update({ lockedAt: db.fn.now() });
} catch (error) {
throw new DatabaseError({ error, name: "EventOutbox: extendClaims" });
}
};
+ // Returns how many rows were settled, which is short of what the caller claimed when the claim was
+ // recycled mid-delivery and the rows now belong to another worker.
const commitResults = async (input: {
+ lockToken: string;
delivered: { ids: string[]; progress?: Record<string, unknown> | null }[];
retriable: { ids: string[]; nextRetryDelayMs: number; progress?: Record<string, unknown> | null; error?: string }[];
failed: { ids: string[]; error?: string }[];
- }): Promise<void> => {
+ }): Promise<number> => {
const total =
input.delivered.reduce((n, g) => n + g.ids.length, 0) +
input.retriable.reduce((n, g) => n + g.ids.length, 0) +
input.failed.reduce((n, g) => n + g.ids.length, 0);
- if (total === 0) return;
+ if (total === 0) return 0;
try {
- await db.transaction(async (tx) => {
- const settle = (ids: string[], values: Record<string, unknown>) =>
- tx(TableName.EventOutbox)
+ return await db.transaction(async (tx) => {
+ let settled = 0;
+ const settle = async (ids: string[], values: Record<string, unknown>) => {
+ settled += await tx(TableName.EventOutbox)
.whereIn("id", ids)
.where("status", EventOutboxStatus.Processing)
- .update({ lockedAt: null, ...values });
+ .where("lockToken", input.lockToken)
+ .update({ lockedAt: null, lockToken: null, ...values });
+ };
const progressOf = (group: { progress?: Record<string, unknown> | null }) =>
group.progress !== undefined ? { progress: JSON.stringify(group.progress) } : {};
@@ -145,6 +164,8 @@ export const eventOutboxDALFactory = (db: TDbClient) => {
lastError: group.error ?? null
});
}
+
+ return settled;
});
} catch (error) {
throw new DatabaseError({ error, name: "EventOutbox: commitResults" });
@@ -188,6 +209,7 @@ export const eventOutboxDALFactory = (db: TDbClient) => {
attempts: db.raw('"attempts" + 1'),
nextRetryAt: db.raw(`NOW() + (? || ' milliseconds')::INTERVAL`, [computeBackoffMs(attempts + 1)]),
lockedAt: null,
+ lockToken: null,
lastError: "Worker did not report a result before the claim went stale"
});
}
@@ -199,6 +221,7 @@ export const eventOutboxDALFactory = (db: TDbClient) => {
status: EventOutboxStatus.Failed,
attempts: db.raw('"attempts" + 1'),
lockedAt: null,
+ lockToken: null,
lastError: "Worker did not report a result before the claim went stale"
});
}
diff --git a/backend/src/services/event-outbox/event-outbox-service.test.ts b/backend/src/services/event-outbox/event-outbox-service.test.ts
index ebbbe7f825..6bf8cdc5b9 100644
--- a/backend/src/services/event-outbox/event-outbox-service.test.ts
+++ b/backend/src/services/event-outbox/event-outbox-service.test.ts
@@ -12,8 +12,9 @@ import {
TOutboxEvent
} from "./event-outbox-types";
+const { loggerWarn } = vi.hoisted(() => ({ loggerWarn: vi.fn<(...args: unknown[]) => void>() }));
vi.mock("@app/lib/logger", () => ({
- logger: { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} },
+ logger: { info: () => {}, warn: loggerWarn, error: () => {}, debug: () => {} },
initLogger: () => {}
}));
@@ -171,6 +172,7 @@ describe("event outbox drain", () => {
batches: TEventOutbox[][];
handle: IEventOutboxConsumer["handle"];
consumerName?: string;
+ settled?: (input: { delivered: { ids: string[] }[] }) => number;
}) => {
const registry = eventOutboxRegistryFactory();
registry.register(makeConsumer({ name: opts.consumerName ?? "alert", handle: opts.handle }));
@@ -180,27 +182,40 @@ describe("event outbox drain", () => {
retriable: { ids: string[]; nextRetryDelayMs: number; error?: string }[];
failed: { ids: string[]; error?: string }[];
}[] = [];
- const extended: string[][] = [];
+ const extended: { ids: string[]; lockToken: string }[] = [];
+ const claimTokens: string[] = [];
let batchIdx = 0;
const service = eventOutboxServiceFactory({
eventOutboxDAL: {
claimBatch: async () => {
- const batch = opts.batches[batchIdx] ?? [];
+ const rows = opts.batches[batchIdx] ?? [];
batchIdx += 1;
- return batch;
+ const lockToken = `token-${batchIdx}`;
+ claimTokens.push(lockToken);
+ return { lockToken, rows };
},
- extendClaims: async (ids: string[]) => {
- extended.push(ids);
+ extendClaims: async (ids: string[], lockToken: string) => {
+ extended.push({ ids, lockToken });
},
commitResults: async (input: never) => {
commits.push(input);
+ const counted = input as unknown as {
+ delivered: { ids: string[] }[];
+ retriable: { ids: string[] }[];
+ failed: { ids: string[] }[];
+ };
+ if (opts.settled) return opts.settled(counted);
+ return [...counted.delivered, ...counted.retriable, ...counted.failed].reduce(
+ (n, group) => n + group.ids.length,
+ 0
+ );
}
} as never,
eventOutboxRegistry: registry
});
- return { service, commits, extended };
+ return { service, commits, extended, claimTokens };
};
const KEY = { consumer: "alert", resourceType: "approval.workflow", resourceId: "policy-1" };
@@ -296,13 +311,54 @@ describe("event outbox drain", () => {
expect(commits[0].delivered.map((group) => group.ids)).toEqual([["1"], ["2"]]);
});
+ // The token the rows were claimed under is what commitResults fences on, so a claim that reports
+ // under a different one would write to whatever worker holds the rows now.
+ test("commits under the token the batch was claimed with", async () => {
+ const { service, commits, claimTokens } = buildDrain({
+ batches: [[makeRow()]],
+ handle: async (rows) => rows.map((row) => ({ id: String(row.id), status: EventOutboxStatus.Delivered }))
+ });
+
+ await service.drain(KEY);
+
+ expect((commits[0] as unknown as { lockToken: string }).lockToken).toBe(claimTokens[0]);
+ });
+
+ // A short settle means the sweeper handed these rows to another worker while this one was still
+ // delivering, so the batch went out twice. Nothing else in the system can see that happened.
+ test("reports a claim that was taken away mid-delivery", async () => {
+ loggerWarn.mockClear();
+ const { service } = buildDrain({
+ batches: [[makeRow({ id: 1 }), makeRow({ id: 2 })]],
+ handle: async (rows) => rows.map((row) => ({ id: String(row.id), status: EventOutboxStatus.Delivered })),
+ settled: () => 1
+ });
+
+ await service.drain(KEY);
+
+ expect(loggerWarn).toHaveBeenCalledTimes(1);
+ expect(loggerWarn.mock.calls[0][0]).toContain("settled 1 of 2 row(s)");
+ });
+
+ test("says nothing when every claimed row settles", async () => {
+ loggerWarn.mockClear();
+ const { service } = buildDrain({
+ batches: [[makeRow()]],
+ handle: async (rows) => rows.map((row) => ({ id: String(row.id), status: EventOutboxStatus.Delivered }))
+ });
+
+ await service.drain(KEY);
+
+ expect(loggerWarn).not.toHaveBeenCalled();
+ });
+
// A batch that legitimately outlives the sweeper's threshold (a slow webhook, many rows) has to
// keep its claim fresh, or the sweeper hands its rows to a second worker mid-delivery.
test("extends the claim while the consumer is still handling a batch", async () => {
vi.useFakeTimers();
try {
let release: () => void = () => {};
- const { service, extended } = buildDrain({
+ const { service, extended, claimTokens } = buildDrain({
batches: [[makeRow({ id: 1 }), makeRow({ id: 2 })]],
handle: (rows) =>
new Promise((resolve) => {
@@ -314,7 +370,7 @@ describe("event outbox drain", () => {
const draining = service.drain(KEY);
await vi.advanceTimersByTimeAsync(10 * 60_000);
expect(extended.length).toBeGreaterThanOrEqual(3);
- expect(extended[0]).toEqual(["1", "2"]);
+ expect(extended[0]).toEqual({ ids: ["1", "2"], lockToken: claimTokens[0] });
release();
await draining;
@@ -357,14 +413,15 @@ describe("event outbox drain", () => {
const service = eventOutboxServiceFactory({
eventOutboxDAL: {
claimBatch: async () => {
- if (batchServed) return [];
+ if (batchServed) return { lockToken: "token-1", rows: [] };
batchServed = true;
- return [makeRow()];
+ return { lockToken: "token-1", rows: [makeRow()] };
},
extendClaims: async () => {},
commitResults: async () => {
commitCalls += 1;
if (commitCalls === 1) throw new Error("connection reset");
+ return 1;
}
} as never,
eventOutboxRegistry: registry
@@ -392,7 +449,7 @@ describe("event outbox drain", () => {
let commitCalls = 0;
const service = eventOutboxServiceFactory({
eventOutboxDAL: {
- claimBatch: async () => [makeRow()],
+ claimBatch: async () => ({ lockToken: "token-1", rows: [makeRow()] }),
extendClaims: async () => {},
commitResults: async () => {
commitCalls += 1;
diff --git a/backend/src/services/event-outbox/event-outbox-service.ts b/backend/src/services/event-outbox/event-outbox-service.ts
index ed90bcc222..12d3058b02 100644
--- a/backend/src/services/event-outbox/event-outbox-service.ts
+++ b/backend/src/services/event-outbox/event-outbox-service.ts
@@ -125,12 +125,11 @@ export const eventOutboxServiceFactory = ({ eventOutboxDAL, eventOutboxRegistry
const $commitWithRetry = async (
{ consumer, resourceType, resourceId }: TOutboxFlushKey,
input: Parameters<TEventOutboxDALFactory["commitResults"]>[0]
- ): Promise<void> => {
+ ): Promise<number> => {
for (let attempt = 1; ; attempt += 1) {
try {
// eslint-disable-next-line no-await-in-loop -- retrying the same statement is the point
- await eventOutboxDAL.commitResults(input);
- return;
+ return await eventOutboxDAL.commitResults(input);
} catch (error) {
if (attempt >= COMMIT_ATTEMPTS) throw error;
logger.warn(
@@ -143,7 +142,7 @@ export const eventOutboxServiceFactory = ({ eventOutboxDAL, eventOutboxRegistry
}
};
- const $applyResults = async (rows: TEventOutbox[], results: TOutboxRowResult[]): Promise<void> => {
+ const $applyResults = async (rows: TEventOutbox[], lockToken: string, results: TOutboxRowResult[]): Promise<void> => {
const byId = new Map(results.map((result) => [result.id, result]));
const now = Date.now();
@@ -191,7 +190,8 @@ export const eventOutboxServiceFactory = ({ eventOutboxDAL, eventOutboxRegistry
});
}
- await $commitWithRetry(rows[0], {
+ const settled = await $commitWithRetry(rows[0], {
+ lockToken,
delivered: groupByOutcome(delivered, (item) => JSON.stringify(item.progress ?? null)),
retriable: groupByOutcome(retriable, (item) =>
JSON.stringify([item.nextRetryDelayMs, item.error ?? null, item.progress ?? null])
@@ -199,6 +199,13 @@ export const eventOutboxServiceFactory = ({ eventOutboxDAL, eventOutboxRegistry
failed: groupByOutcome(failed, (item) => item.error ?? "")
});
+ if (settled < rows.length) {
+ const { consumer, resourceType, resourceId } = rows[0];
+ logger.warn(
+ `event-outbox: settled ${settled} of ${rows.length} row(s); the rest were no longer held by this claim [consumer=${consumer}] [resourceType=${resourceType}] [resourceId=${resourceId}]`
+ );
+ }
+
if (failed.length > 0) {
const { consumer, resourceType, resourceId } = rows[0];
recordEventOutboxExhaustedMetric({ consumer, count: failed.length });
@@ -213,11 +220,12 @@ export const eventOutboxServiceFactory = ({ eventOutboxDAL, eventOutboxRegistry
const $handleClaimed = async (
consumer: IEventOutboxConsumer,
key: TOutboxFlushKey,
- claimed: TEventOutbox[]
+ claimed: TEventOutbox[],
+ lockToken: string
): Promise<TOutboxRowResult[]> => {
const ids = claimed.map((row) => String(row.id));
const heartbeat = setInterval(() => {
- eventOutboxDAL.extendClaims(ids).catch((error) => {
+ eventOutboxDAL.extendClaims(ids, lockToken).catch((error) => {
logger.warn(
error,
`event-outbox: failed to extend a claim [consumer=${key.consumer}] [resourceType=${key.resourceType}] [resourceId=${key.resourceId}]`
@@ -253,13 +261,13 @@ export const eventOutboxServiceFactory = ({ eventOutboxDAL, eventOutboxRegistry
for (let batch = 0; batch < MAX_BATCHES_PER_FLUSH; batch += 1) {
// eslint-disable-next-line no-await-in-loop -- batches are serial on purpose, to stay paced
- const claimed = await eventOutboxDAL.claimBatch(key, OUTBOX_CLAIM_BATCH_SIZE);
+ const { lockToken, rows: claimed } = await eventOutboxDAL.claimBatch(key, OUTBOX_CLAIM_BATCH_SIZE);
if (claimed.length === 0) break;
// eslint-disable-next-line no-await-in-loop -- see above
- const results = await $handleClaimed(consumer, key, claimed);
+ const results = await $handleClaimed(consumer, key, claimed, lockToken);
// eslint-disable-next-line no-await-in-loop -- see above
- await $applyResults(claimed, results);
+ await $applyResults(claimed, lockToken, results);
handled += claimed.length;
}