Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.
TypeScript acmeagentsai-agentscertificate-managementdevsecopsenvironment-variableskubernetesopen-sourcepampkipostgresprivate-caprivileged-access-managementsecret-managementsecret-managersecretssecrets-managementsecuritysecurity-toolsvault
external limit on sweep
Thiago Araujo da Silvac635702e6a69
diff --git a/backend/e2e-test/event-outbox.spec.ts b/backend/e2e-test/event-outbox.spec.tsindex 3b70750e5d..d50d7c6906 100644--- a/backend/e2e-test/event-outbox.spec.ts+++ b/backend/e2e-test/event-outbox.spec.ts@@ -137,9 +137,13 @@ describe("event outbox (postgres)", () => {.where("id", exhausted.id).update({ lockedAt: longAgo, attempts: MAX_OUTBOX_ATTEMPTS - 1 });- const outcome = await dal.recoverStaleClaims(10 * 60_000, MAX_OUTBOX_ATTEMPTS);+ const outcome = await dal.recoverStaleClaims({+ thresholdMs: 10 * 60_000,+ maxAttempts: MAX_OUTBOX_ATTEMPTS,+ limit: 100+ });- expect(outcome).toEqual({ retried: 1, failed: 1 });+ expect(outcome).toEqual({ retried: 1, failed: [{ consumer: CONSUMER, count: 1 }] });const [untouched, retried, failed] = await rowsFor();expect(untouched.status).toBe(EventOutboxStatus.Processing);expect(retried.status).toBe(EventOutboxStatus.Retry);diff --git a/backend/src/services/event-outbox/event-outbox-dal.test.ts b/backend/src/services/event-outbox/event-outbox-dal.test.tsindex cc4febbab1..38f8a6ec23 100644--- a/backend/src/services/event-outbox/event-outbox-dal.test.ts+++ b/backend/src/services/event-outbox/event-outbox-dal.test.ts@@ -277,13 +277,26 @@ describe("event outbox dal", () => {test("recoverStaleClaims looks only at claims older than the threshold", async () => {const { dal, calls } = buildDAL();- await dal.recoverStaleClaims(600_000, 5);+ await dal.recoverStaleClaims({ thresholdMs: 600_000, maxAttempts: 5, limit: 1_000 });expect(calls.where).toContainEqual(["status", EventOutboxStatus.Processing]);expect(calls.andWhereRaw[0][0]).toBe(`"lockedAt" < NOW() - (? || ' milliseconds')::INTERVAL`);expect(calls.andWhereRaw[0][1]).toEqual([600_000]);});+ // The transaction bounds itself rather than trusting worker concurrency to keep `processing` small,+ // and takes the oldest claims first so a limit never starves one.+ test("recoverStaleClaims recovers a bounded batch, oldest claim first", async () => {+ const { dal, calls } = buildDAL();++ await dal.recoverStaleClaims({ thresholdMs: 600_000, maxAttempts: 5, limit: 1_000 });++ expect(calls.limit).toContain(1_000);+ expect(calls.orderBy).toContainEqual(["lockedAt", "asc"]);+ expect(calls.forUpdate).toBe(1);+ expect(calls.skipLocked).toBe(1);+ });+test("prune targets the outbox table", async () => {const { dal } = buildDAL();diff --git a/backend/src/services/event-outbox/event-outbox-dal.ts b/backend/src/services/event-outbox/event-outbox-dal.tsindex a0ba0d1303..deaeffc7a5 100644--- a/backend/src/services/event-outbox/event-outbox-dal.ts+++ b/backend/src/services/event-outbox/event-outbox-dal.ts@@ -161,23 +161,27 @@ export const eventOutboxDALFactory = (db: TDbClient) => {}};- const recoverStaleClaims = async (- thresholdMs: number,- maxAttempts: number- ): Promise<{ retried: number; failed: number }> => {+ const recoverStaleClaims = async (input: {+ thresholdMs: number;+ maxAttempts: number;+ limit: number;+ }): Promise<{ retried: number; failed: { consumer: string; count: number }[] }> => {try {return await db.transaction(async (tx) => {const staleRows = await tx(TableName.EventOutbox).where("status", EventOutboxStatus.Processing)- .andWhereRaw(`"lockedAt" < NOW() - (? || ' milliseconds')::INTERVAL`, [thresholdMs])+ .andWhereRaw(`"lockedAt" < NOW() - (? || ' milliseconds')::INTERVAL`, [input.thresholdMs])+ .orderBy("lockedAt", "asc")+ .limit(input.limit).forUpdate().skipLocked()- .select<{ id: string; attempts: number }[]>("id", "attempts");+ .select<{ id: string; consumer: string; attempts: number }[]>("id", "consumer", "attempts");- if (staleRows.length === 0) return { retried: 0, failed: 0 };+ if (staleRows.length === 0) return { retried: 0, failed: [] };- const exhausted = staleRows.filter((row) => row.attempts + 1 >= maxAttempts).map((row) => row.id);- const retriable = staleRows.filter((row) => row.attempts + 1 < maxAttempts);+ const exhaustedRows = staleRows.filter((row) => row.attempts + 1 >= input.maxAttempts);+ const exhausted = exhaustedRows.map((row) => row.id);+ const retriable = staleRows.filter((row) => row.attempts + 1 < input.maxAttempts);const retriableByAttempt = new Map<number, string[]>();for (const row of retriable) {@@ -209,7 +213,15 @@ export const eventOutboxDALFactory = (db: TDbClient) => {});}- return { retried: retriable.length, failed: exhausted.length };+ const failedByConsumer = new Map<string, number>();+ for (const row of exhaustedRows) {+ failedByConsumer.set(row.consumer, (failedByConsumer.get(row.consumer) ?? 0) + 1);+ }++ return {+ retried: retriable.length,+ failed: [...failedByConsumer].map(([consumer, count]) => ({ consumer, count }))+ };});} catch (error) {throw new DatabaseError({ error, name: "EventOutbox: recoverStaleClaims" });diff --git a/backend/src/services/event-outbox/event-outbox-service.test.ts b/backend/src/services/event-outbox/event-outbox-service.test.tsindex c3945ae8fb..dc104998e7 100644--- a/backend/src/services/event-outbox/event-outbox-service.test.ts+++ b/backend/src/services/event-outbox/event-outbox-service.test.ts@@ -16,6 +16,14 @@ vi.mock("@app/lib/logger", () => ({initLogger: () => {}}));+const { exhaustedMetrics } = vi.hoisted(() => ({ exhaustedMetrics: [] as { consumer: string; count: number }[] }));+vi.mock("@app/lib/telemetry/metrics", () => ({+ recordEventOutboxLagMetric: () => {},+ recordEventOutboxExhaustedMetric: (params: { consumer: string; count: number }) => {+ exhaustedMetrics.push(params);+ }+}));+const ORG_ID = "11111111-1111-1111-1111-111111111111";const makeEvent = (overrides?: Partial<TOutboxEvent>): TOutboxEvent => ({@@ -404,3 +412,76 @@ describe("event outbox drain", () => {}});});++describe("event outbox stale sweep", () => {+ const buildSweep = (batches: { retried: number; failed: { consumer: string; count: number }[] }[]) => {+ const calls: { limit: number }[] = [];+ const service = eventOutboxServiceFactory({+ eventOutboxDAL: {+ recoverStaleClaims: async (input: { limit: number }) => {+ calls.push(input);+ return batches[calls.length - 1] ?? { retried: 0, failed: [] };+ }+ } as never,+ eventOutboxRegistry: eventOutboxRegistryFactory()+ });+ return { service, calls };+ };++ beforeEach(() => {+ exhaustedMetrics.length = 0;+ });++ // A row the sweeper gives up on is as lost as one a consumer gives up on. Without this the metric+ // that promises to count every abandoned notification misses the ones caused by dead workers.+ test("counts events the sweeper gives up on, per consumer, on the exhausted metric", async () => {+ const { service } = buildSweep([+ {+ retried: 3,+ failed: [+ { consumer: "alert", count: 2 },+ { consumer: "audit", count: 1 }+ ]+ }+ ]);++ await service.sweepStaleClaims();++ expect(exhaustedMetrics).toEqual([+ { consumer: "alert", count: 2 },+ { consumer: "audit", count: 1 }+ ]);+ });++ test("records nothing when the sweep found no stale claims", async () => {+ const { service, calls } = buildSweep([]);++ await service.sweepStaleClaims();++ expect(calls).toHaveLength(1);+ expect(exhaustedMetrics).toEqual([]);+ });++ test("keeps sweeping while batches come back full, and stops at a bound", async () => {+ const { service, calls } = buildSweep(+ Array.from({ length: 50 }, () => ({ retried: 1_000, failed: [] as { consumer: string; count: number }[] }))+ );++ await service.sweepStaleClaims();++ expect(calls).toHaveLength(10);+ expect(calls.every((call) => call.limit === 1_000)).toBe(true);+ });++ test("stops after the first batch that comes back short", async () => {+ const { service, calls } = buildSweep([+ { retried: 1_000, failed: [] },+ { retried: 998, failed: [{ consumer: "alert", count: 1 }] },+ { retried: 1_000, failed: [] }+ ]);++ await service.sweepStaleClaims();++ expect(calls).toHaveLength(2);+ });+});diff --git a/backend/src/services/event-outbox/event-outbox-service.ts b/backend/src/services/event-outbox/event-outbox-service.tsindex 2953de63f9..4bcdc37d03 100644--- a/backend/src/services/event-outbox/event-outbox-service.ts+++ b/backend/src/services/event-outbox/event-outbox-service.ts@@ -27,6 +27,9 @@ import {const PRUNE_BATCH_SIZE = 5_000;const PRUNE_MAX_BATCHES = 20;+const STALE_SWEEP_BATCH_SIZE = 1_000;+const STALE_SWEEP_MAX_BATCHES = 10;+// Frequent enough that one missed beat (a slow query, a paused event loop) doesn't let the sweeper// call a live claim stale.const CLAIM_HEARTBEAT_INTERVAL_MS = STALE_CLAIM_THRESHOLD_MS / 4;@@ -276,7 +279,33 @@ export const eventOutboxServiceFactory = ({ eventOutboxDAL, eventOutboxRegistry};const sweepStaleClaims = async (): Promise<void> => {- const { retried, failed } = await eventOutboxDAL.recoverStaleClaims(STALE_CLAIM_THRESHOLD_MS, MAX_OUTBOX_ATTEMPTS);+ let retried = 0;+ const failedByConsumer = new Map<string, number>();++ for (let batch = 0; batch < STALE_SWEEP_MAX_BATCHES; batch += 1) {+ // eslint-disable-next-line no-await-in-loop -- batches must run serially to stay paced+ const outcome = await eventOutboxDAL.recoverStaleClaims({+ thresholdMs: STALE_CLAIM_THRESHOLD_MS,+ maxAttempts: MAX_OUTBOX_ATTEMPTS,+ limit: STALE_SWEEP_BATCH_SIZE+ });+ retried += outcome.retried;+ let failedInBatch = 0;+ for (const { consumer, count } of outcome.failed) {+ failedInBatch += count;+ failedByConsumer.set(consumer, (failedByConsumer.get(consumer) ?? 0) + count);+ }+ if (outcome.retried + failedInBatch < STALE_SWEEP_BATCH_SIZE) break;+ }++ for (const [consumer, count] of failedByConsumer) {+ recordEventOutboxExhaustedMetric({ consumer, count });+ logger.error(+ `event-outbox: gave up on ${count} event(s) whose worker never reported back [consumer=${consumer}] [thresholdMs=${STALE_CLAIM_THRESHOLD_MS}]`+ );+ }++ const failed = [...failedByConsumer.values()].reduce((sum, count) => sum + count, 0);if (retried > 0 || failed > 0) {logger.warn(`event-outbox: recovered stale claims [retried=${retried}] [failed=${failed}] [thresholdMs=${STALE_CLAIM_THRESHOLD_MS}]`