Skip to content

Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.

external limit on sweep

Thiago Araujo da Silvac635702e6a69
diff --git a/backend/e2e-test/event-outbox.spec.ts b/backend/e2e-test/event-outbox.spec.ts
index 3b70750e5d..d50d7c6906 100644
--- a/backend/e2e-test/event-outbox.spec.ts
+++ b/backend/e2e-test/event-outbox.spec.ts
@@ -137,9 +137,13 @@ describe("event outbox (postgres)", () => {
.where("id", exhausted.id)
.update({ lockedAt: longAgo, attempts: MAX_OUTBOX_ATTEMPTS - 1 });
- const outcome = await dal.recoverStaleClaims(10 * 60_000, MAX_OUTBOX_ATTEMPTS);
+ const outcome = await dal.recoverStaleClaims({
+ thresholdMs: 10 * 60_000,
+ maxAttempts: MAX_OUTBOX_ATTEMPTS,
+ limit: 100
+ });
- expect(outcome).toEqual({ retried: 1, failed: 1 });
+ expect(outcome).toEqual({ retried: 1, failed: [{ consumer: CONSUMER, count: 1 }] });
const [untouched, retried, failed] = await rowsFor();
expect(untouched.status).toBe(EventOutboxStatus.Processing);
expect(retried.status).toBe(EventOutboxStatus.Retry);
diff --git a/backend/src/services/event-outbox/event-outbox-dal.test.ts b/backend/src/services/event-outbox/event-outbox-dal.test.ts
index cc4febbab1..38f8a6ec23 100644
--- a/backend/src/services/event-outbox/event-outbox-dal.test.ts
+++ b/backend/src/services/event-outbox/event-outbox-dal.test.ts
@@ -277,13 +277,26 @@ describe("event outbox dal", () => {
test("recoverStaleClaims looks only at claims older than the threshold", async () => {
const { dal, calls } = buildDAL();
- await dal.recoverStaleClaims(600_000, 5);
+ await dal.recoverStaleClaims({ thresholdMs: 600_000, maxAttempts: 5, limit: 1_000 });
expect(calls.where).toContainEqual(["status", EventOutboxStatus.Processing]);
expect(calls.andWhereRaw[0][0]).toBe(`"lockedAt" < NOW() - (? || ' milliseconds')::INTERVAL`);
expect(calls.andWhereRaw[0][1]).toEqual([600_000]);
});
+ // The transaction bounds itself rather than trusting worker concurrency to keep `processing` small,
+ // and takes the oldest claims first so a limit never starves one.
+ test("recoverStaleClaims recovers a bounded batch, oldest claim first", async () => {
+ const { dal, calls } = buildDAL();
+
+ await dal.recoverStaleClaims({ thresholdMs: 600_000, maxAttempts: 5, limit: 1_000 });
+
+ expect(calls.limit).toContain(1_000);
+ expect(calls.orderBy).toContainEqual(["lockedAt", "asc"]);
+ expect(calls.forUpdate).toBe(1);
+ expect(calls.skipLocked).toBe(1);
+ });
+
test("prune targets the outbox table", async () => {
const { dal } = buildDAL();
diff --git a/backend/src/services/event-outbox/event-outbox-dal.ts b/backend/src/services/event-outbox/event-outbox-dal.ts
index a0ba0d1303..deaeffc7a5 100644
--- a/backend/src/services/event-outbox/event-outbox-dal.ts
+++ b/backend/src/services/event-outbox/event-outbox-dal.ts
@@ -161,23 +161,27 @@ export const eventOutboxDALFactory = (db: TDbClient) => {
}
};
- const recoverStaleClaims = async (
- thresholdMs: number,
- maxAttempts: number
- ): Promise<{ retried: number; failed: number }> => {
+ const recoverStaleClaims = async (input: {
+ thresholdMs: number;
+ maxAttempts: number;
+ limit: number;
+ }): Promise<{ retried: number; failed: { consumer: string; count: number }[] }> => {
try {
return await db.transaction(async (tx) => {
const staleRows = await tx(TableName.EventOutbox)
.where("status", EventOutboxStatus.Processing)
- .andWhereRaw(`"lockedAt" < NOW() - (? || ' milliseconds')::INTERVAL`, [thresholdMs])
+ .andWhereRaw(`"lockedAt" < NOW() - (? || ' milliseconds')::INTERVAL`, [input.thresholdMs])
+ .orderBy("lockedAt", "asc")
+ .limit(input.limit)
.forUpdate()
.skipLocked()
- .select<{ id: string; attempts: number }[]>("id", "attempts");
+ .select<{ id: string; consumer: string; attempts: number }[]>("id", "consumer", "attempts");
- if (staleRows.length === 0) return { retried: 0, failed: 0 };
+ if (staleRows.length === 0) return { retried: 0, failed: [] };
- const exhausted = staleRows.filter((row) => row.attempts + 1 >= maxAttempts).map((row) => row.id);
- const retriable = staleRows.filter((row) => row.attempts + 1 < maxAttempts);
+ const exhaustedRows = staleRows.filter((row) => row.attempts + 1 >= input.maxAttempts);
+ const exhausted = exhaustedRows.map((row) => row.id);
+ const retriable = staleRows.filter((row) => row.attempts + 1 < input.maxAttempts);
const retriableByAttempt = new Map<number, string[]>();
for (const row of retriable) {
@@ -209,7 +213,15 @@ export const eventOutboxDALFactory = (db: TDbClient) => {
});
}
- return { retried: retriable.length, failed: exhausted.length };
+ const failedByConsumer = new Map<string, number>();
+ for (const row of exhaustedRows) {
+ failedByConsumer.set(row.consumer, (failedByConsumer.get(row.consumer) ?? 0) + 1);
+ }
+
+ return {
+ retried: retriable.length,
+ failed: [...failedByConsumer].map(([consumer, count]) => ({ consumer, count }))
+ };
});
} catch (error) {
throw new DatabaseError({ error, name: "EventOutbox: recoverStaleClaims" });
diff --git a/backend/src/services/event-outbox/event-outbox-service.test.ts b/backend/src/services/event-outbox/event-outbox-service.test.ts
index c3945ae8fb..dc104998e7 100644
--- a/backend/src/services/event-outbox/event-outbox-service.test.ts
+++ b/backend/src/services/event-outbox/event-outbox-service.test.ts
@@ -16,6 +16,14 @@ vi.mock("@app/lib/logger", () => ({
initLogger: () => {}
}));
+const { exhaustedMetrics } = vi.hoisted(() => ({ exhaustedMetrics: [] as { consumer: string; count: number }[] }));
+vi.mock("@app/lib/telemetry/metrics", () => ({
+ recordEventOutboxLagMetric: () => {},
+ recordEventOutboxExhaustedMetric: (params: { consumer: string; count: number }) => {
+ exhaustedMetrics.push(params);
+ }
+}));
+
const ORG_ID = "11111111-1111-1111-1111-111111111111";
const makeEvent = (overrides?: Partial<TOutboxEvent>): TOutboxEvent => ({
@@ -404,3 +412,76 @@ describe("event outbox drain", () => {
}
});
});
+
+describe("event outbox stale sweep", () => {
+ const buildSweep = (batches: { retried: number; failed: { consumer: string; count: number }[] }[]) => {
+ const calls: { limit: number }[] = [];
+ const service = eventOutboxServiceFactory({
+ eventOutboxDAL: {
+ recoverStaleClaims: async (input: { limit: number }) => {
+ calls.push(input);
+ return batches[calls.length - 1] ?? { retried: 0, failed: [] };
+ }
+ } as never,
+ eventOutboxRegistry: eventOutboxRegistryFactory()
+ });
+ return { service, calls };
+ };
+
+ beforeEach(() => {
+ exhaustedMetrics.length = 0;
+ });
+
+ // A row the sweeper gives up on is as lost as one a consumer gives up on. Without this the metric
+ // that promises to count every abandoned notification misses the ones caused by dead workers.
+ test("counts events the sweeper gives up on, per consumer, on the exhausted metric", async () => {
+ const { service } = buildSweep([
+ {
+ retried: 3,
+ failed: [
+ { consumer: "alert", count: 2 },
+ { consumer: "audit", count: 1 }
+ ]
+ }
+ ]);
+
+ await service.sweepStaleClaims();
+
+ expect(exhaustedMetrics).toEqual([
+ { consumer: "alert", count: 2 },
+ { consumer: "audit", count: 1 }
+ ]);
+ });
+
+ test("records nothing when the sweep found no stale claims", async () => {
+ const { service, calls } = buildSweep([]);
+
+ await service.sweepStaleClaims();
+
+ expect(calls).toHaveLength(1);
+ expect(exhaustedMetrics).toEqual([]);
+ });
+
+ test("keeps sweeping while batches come back full, and stops at a bound", async () => {
+ const { service, calls } = buildSweep(
+ Array.from({ length: 50 }, () => ({ retried: 1_000, failed: [] as { consumer: string; count: number }[] }))
+ );
+
+ await service.sweepStaleClaims();
+
+ expect(calls).toHaveLength(10);
+ expect(calls.every((call) => call.limit === 1_000)).toBe(true);
+ });
+
+ test("stops after the first batch that comes back short", async () => {
+ const { service, calls } = buildSweep([
+ { retried: 1_000, failed: [] },
+ { retried: 998, failed: [{ consumer: "alert", count: 1 }] },
+ { retried: 1_000, failed: [] }
+ ]);
+
+ await service.sweepStaleClaims();
+
+ expect(calls).toHaveLength(2);
+ });
+});
diff --git a/backend/src/services/event-outbox/event-outbox-service.ts b/backend/src/services/event-outbox/event-outbox-service.ts
index 2953de63f9..4bcdc37d03 100644
--- a/backend/src/services/event-outbox/event-outbox-service.ts
+++ b/backend/src/services/event-outbox/event-outbox-service.ts
@@ -27,6 +27,9 @@ import {
const PRUNE_BATCH_SIZE = 5_000;
const PRUNE_MAX_BATCHES = 20;
+const STALE_SWEEP_BATCH_SIZE = 1_000;
+const STALE_SWEEP_MAX_BATCHES = 10;
+
// Frequent enough that one missed beat (a slow query, a paused event loop) doesn't let the sweeper
// call a live claim stale.
const CLAIM_HEARTBEAT_INTERVAL_MS = STALE_CLAIM_THRESHOLD_MS / 4;
@@ -276,7 +279,33 @@ export const eventOutboxServiceFactory = ({ eventOutboxDAL, eventOutboxRegistry
};
const sweepStaleClaims = async (): Promise<void> => {
- const { retried, failed } = await eventOutboxDAL.recoverStaleClaims(STALE_CLAIM_THRESHOLD_MS, MAX_OUTBOX_ATTEMPTS);
+ let retried = 0;
+ const failedByConsumer = new Map<string, number>();
+
+ for (let batch = 0; batch < STALE_SWEEP_MAX_BATCHES; batch += 1) {
+ // eslint-disable-next-line no-await-in-loop -- batches must run serially to stay paced
+ const outcome = await eventOutboxDAL.recoverStaleClaims({
+ thresholdMs: STALE_CLAIM_THRESHOLD_MS,
+ maxAttempts: MAX_OUTBOX_ATTEMPTS,
+ limit: STALE_SWEEP_BATCH_SIZE
+ });
+ retried += outcome.retried;
+ let failedInBatch = 0;
+ for (const { consumer, count } of outcome.failed) {
+ failedInBatch += count;
+ failedByConsumer.set(consumer, (failedByConsumer.get(consumer) ?? 0) + count);
+ }
+ if (outcome.retried + failedInBatch < STALE_SWEEP_BATCH_SIZE) break;
+ }
+
+ for (const [consumer, count] of failedByConsumer) {
+ recordEventOutboxExhaustedMetric({ consumer, count });
+ logger.error(
+ `event-outbox: gave up on ${count} event(s) whose worker never reported back [consumer=${consumer}] [thresholdMs=${STALE_CLAIM_THRESHOLD_MS}]`
+ );
+ }
+
+ const failed = [...failedByConsumer.values()].reduce((sum, count) => sum + count, 0);
if (retried > 0 || failed > 0) {
logger.warn(
`event-outbox: recovered stale claims [retried=${retried}] [failed=${failed}] [thresholdMs=${STALE_CLAIM_THRESHOLD_MS}]`