Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.
TypeScript acmeagentsai-agentscertificate-managementdevsecopsenvironment-variableskubernetesopen-sourcepampkipostgresprivate-caprivileged-access-managementsecret-managementsecret-managersecretssecrets-managementsecuritysecurity-toolsvault
timeout to avoid heartbit extending the claim forever
Thiago Araujo da Silva28846d297987
diff --git a/backend/src/services/event-outbox/event-outbox-service.test.ts b/backend/src/services/event-outbox/event-outbox-service.test.tsindex a1da7088fb..2cce8ceb34 100644--- a/backend/src/services/event-outbox/event-outbox-service.test.ts+++ b/backend/src/services/event-outbox/event-outbox-service.test.ts@@ -362,6 +362,34 @@ describe("event outbox drain", () => {}});+ // Without this, a handle() that never settles keeps the heartbeat alive forever and, because the+ // flush job id is per consumer, blocks every later flush for that consumer until a restart.+ test("gives up on a consumer that never settles and retries the batch", async () => {+ vi.useFakeTimers();+ try {+ const { service, commits, extended } = buildDrain({+ batches: [[makeRow({ id: 1 }), makeRow({ id: 2 })]],+ handle: () => new Promise(() => {})+ });++ const draining = service.drain(KEY);+ await vi.advanceTimersByTimeAsync(30 * 60_000);+ await expect(draining).resolves.toEqual({ handled: 2, unknownConsumer: false });++ expect(commits).toHaveLength(1);+ expect(commits[0].delivered).toEqual([]);+ expect(commits[0].retriable).toHaveLength(1);+ expect(commits[0].retriable[0].ids).toEqual(["1", "2"]);+ expect(commits[0].retriable[0].error).toContain("did not finish handling the batch");++ const afterTimeout = extended.length;+ await vi.advanceTimersByTimeAsync(10 * 60_000);+ expect(extended.length).toBe(afterTimeout);+ } finally {+ vi.useRealTimers();+ }+ });+test("leaves rows untouched when no consumer is registered under the key", async () => {const { service, commits } = buildDrain({ batches: [[makeRow()]], handle: async () => [] });diff --git a/backend/src/services/event-outbox/event-outbox-service.ts b/backend/src/services/event-outbox/event-outbox-service.tsindex cba2df809b..de95abcf17 100644--- a/backend/src/services/event-outbox/event-outbox-service.ts+++ b/backend/src/services/event-outbox/event-outbox-service.ts@@ -34,6 +34,12 @@ const STALE_SWEEP_MAX_BATCHES = 10;// A quarter of the threshold so one missed beat doesn't get a live claim swept.const CLAIM_HEARTBEAT_INTERVAL_MS = STALE_CLAIM_THRESHOLD_MS / 4;+// Deliberately longer than the stale threshold: the heartbeat exists so a legitimately slow batch can+// outlive it. This is the backstop for a handle() that never settles, which would otherwise keep the+// heartbeat, and with it the per-consumer flush job, alive until the process restarts. It cannot+// cancel the work in flight, so an abandoned batch may still deliver and be delivered again on retry.+const CONSUMER_HANDLE_TIMEOUT_MS = 3 * STALE_CLAIM_THRESHOLD_MS;+const COMMIT_ATTEMPTS = 3;const COMMIT_RETRY_DELAY_MS = 250;@@ -42,6 +48,25 @@ const sleep = (ms: number) =>setTimeout(resolve, ms);});+class ConsumerHandleTimeoutError extends Error {+ constructor(consumer: string, ms: number) {+ super(`Consumer '${consumer}' did not finish handling the batch within ${ms / 1000}s`);+ this.name = "ConsumerHandleTimeoutError";+ }+}++const withTimeout = async <T>(promise: Promise<T>, ms: number, makeError: () => Error): Promise<T> => {+ let timer: NodeJS.Timeout | undefined;+ const timeout = new Promise<never>((_, reject) => {+ timer = setTimeout(() => reject(makeError()), ms);+ });+ try {+ return await Promise.race([promise, timeout]);+ } finally {+ if (timer) clearTimeout(timer);+ }+};+const formatIssues = (issues: { path: (string | number)[]; message: string }[]) =>issues.map((issue) => `${issue.path.join(".")} ${issue.message}`).join(", ");@@ -222,10 +247,20 @@ export const eventOutboxServiceFactory = ({ eventOutboxDAL, eventOutboxRegistry}, CLAIM_HEARTBEAT_INTERVAL_MS);try {- return await consumer.handle(claimed);+ return await withTimeout(+ consumer.handle(claimed),+ CONSUMER_HANDLE_TIMEOUT_MS,+ () => new ConsumerHandleTimeoutError(key.consumer, CONSUMER_HANDLE_TIMEOUT_MS)+ );} catch (error) {const message = error instanceof Error ? error.message : "Unknown error";- logger.error(error, `event-outbox: consumer '${key.consumer}' threw while handling a batch`);+ if (error instanceof ConsumerHandleTimeoutError) {+ logger.error(+ `event-outbox: consumer '${key.consumer}' timed out on a batch of ${claimed.length}; retrying it later [timeoutMs=${CONSUMER_HANDLE_TIMEOUT_MS}]`+ );+ } else {+ logger.error(error, `event-outbox: consumer '${key.consumer}' threw while handling a batch`);+ }return claimed.map((row) => ({ id: String(row.id), status: EventResultStatus.Retry, error: message }));} finally {clearInterval(heartbeat);