Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.
TypeScript acmeagentsai-agentscertificate-managementdevsecopsenvironment-variableskubernetesopen-sourcepampkipostgresprivate-caprivileged-access-managementsecret-managementsecret-managersecretssecrets-managementsecuritysecurity-toolsvault
add plimit and insert updatedAt on the idepotency key to avoid retry burst at the same time
adilsonc98a55ad7859
diff --git a/CLAUDE.md b/CLAUDE.mdindex 4f7b46bc22..a74c2aa08d 100644--- a/CLAUDE.md+++ b/CLAUDE.md@@ -169,8 +169,10 @@ failure returns `Retry`, so the outbox backs off and tries again (up to `MAX_OUTpromises that retry on every attempt because the consumer cannot see the attempt count. A user resends a`failed_dispatch` request with `POST /api/v1/access-approvals/requests/:requestId/retry-delegated-dispatch`. Theproduct's bridge checks who may retry, and `delegatedApprovalService.retryDispatch` checks the request can be-retried and emits the same event with no transaction. One flush runs per consumer at a time, so two dispatches-of one request never run together; a second event for a request that was already delivered is marked delivered+retried and emits the same event with no transaction. The emit's idempotency key is the event, the delegated+request id, and that row's `updatedAt`, so repeated emits of one attempt collapse, and a later resend still+inserts after the status write bumps `updatedAt`. A batch dispatches up to 10 different requests at once;+events for one request stay serial. A second event for a request that was already delivered is marked deliveredwithout calling the provider. A delegated request hasno Infisical approvers, so `POST /api/v1/access-approvals/requests/:requestId/cancel-delegated-dispatch` is the only way to closeone from Infisical: the requester, or anyone who can edit the policy, sets it `cancelled`, which is also how adiff --git a/backend/src/services/delegated-approval/delegated-approval-events.ts b/backend/src/services/delegated-approval/delegated-approval-events.tsindex be728eea7d..0e9edc777d 100644--- a/backend/src/services/delegated-approval/delegated-approval-events.ts+++ b/backend/src/services/delegated-approval/delegated-approval-events.ts@@ -13,16 +13,19 @@ export type TDelegatedApprovalDispatchRequestedPayload = z.infer<typeof DelegatedApprovalDispatchRequestedPayloadSchema>;-// No idempotency key on purpose: a user retry has to insert a fresh row while the delivered one from the-// first attempt is still retained, and the consumer already skips a request that was delivered.+// updatedAt is the attempt. Repeated retries of one failure share it and collapse on the outbox unique+// key. A request id alone would not: the previous row is still retained (delivered for a day, failed for+// 30 days), so a later resend has to insert beside it. The consumer's status write bumps updatedAt,+// which is what gives that resend a new key.export const emitDelegatedApprovalDispatchRequested = (eventEmitter: TEventEmitter,- { delegatedRequestId }: TDelegatedApprovalDispatchRequestedPayload,+ { delegatedRequestId, updatedAt }: TDelegatedApprovalDispatchRequestedPayload & { updatedAt: Date },tx?: Knex): Promise<void> =>eventEmitter.emit({eventType: DELEGATED_APPROVAL_DISPATCH_REQUESTED_EVENT,+ idempotencyKey: `${DELEGATED_APPROVAL_DISPATCH_REQUESTED_EVENT}:${delegatedRequestId}:${updatedAt.toISOString()}`,payload: { delegatedRequestId }},txdiff --git a/backend/src/services/delegated-approval/delegated-approval-service.test.ts b/backend/src/services/delegated-approval/delegated-approval-service.test.tsindex 8f3118b4ab..1fc4369419 100644--- a/backend/src/services/delegated-approval/delegated-approval-service.test.ts+++ b/backend/src/services/delegated-approval/delegated-approval-service.test.ts@@ -7,7 +7,13 @@ import { DelegatedApprovalPolicyType, DelegatedApprovalRequestStatus } from "./dimport { DELEGATED_APPROVAL_DISPATCH_REQUESTED_EVENT } from "./delegated-approval-events";import { delegatedApprovalServiceFactory } from "./delegated-approval-service";-const failedRequest = { id: "delegated-1", requestId: "req-1", status: DelegatedApprovalRequestStatus.FailedDispatch };+const UPDATED_AT = new Date("2026-10-23T13:00:00.000Z");+const failedRequest = {+ id: "delegated-1",+ requestId: "req-1",+ status: DelegatedApprovalRequestStatus.FailedDispatch,+ updatedAt: UPDATED_AT+};const serviceNowPolicy = {id: "delegated-policy-1",policyId: "policy-1",@@ -31,6 +37,7 @@ const createService = ({const emit = vi.fn().mockResolvedValue(undefined);const create = vi.fn().mockImplementation(async (row: { requestId: string; status: string }) => ({id: "delegated-1",+ updatedAt: UPDATED_AT,...row}));const service = delegatedApprovalServiceFactory({@@ -43,6 +50,7 @@ const createService = ({const dispatchRequestedEvent = {eventType: DELEGATED_APPROVAL_DISPATCH_REQUESTED_EVENT,+ idempotencyKey: `${DELEGATED_APPROVAL_DISPATCH_REQUESTED_EVENT}:delegated-1:${UPDATED_AT.toISOString()}`,payload: { delegatedRequestId: "delegated-1" }};@@ -73,6 +81,17 @@ describe("delegatedApprovalService.retryDispatch", () => {expect(emit).toHaveBeenCalledWith(dispatchRequestedEvent, undefined);});+ test("emits the same idempotency key when the same failed attempt is sent again", async () => {+ const { service, emit } = createService();++ await service.retryDispatch({ approvalRequest: pendingApprovalRequest });+ await service.retryDispatch({ approvalRequest: pendingApprovalRequest });++ expect(emit).toHaveBeenCalledTimes(2);+ expect(emit).toHaveBeenNthCalledWith(1, dispatchRequestedEvent, undefined);+ expect(emit).toHaveBeenNthCalledWith(2, dispatchRequestedEvent, undefined);+ });+test("refuses a request without a delegated approval", async () => {const { service, emit } = createService({ delegatedRequest: null });diff --git a/backend/src/services/delegated-approval/delegated-approval-service.ts b/backend/src/services/delegated-approval/delegated-approval-service.tsindex e2fe2f857a..0be7efbd01 100644--- a/backend/src/services/delegated-approval/delegated-approval-service.ts+++ b/backend/src/services/delegated-approval/delegated-approval-service.ts@@ -157,7 +157,11 @@ export const delegatedApprovalServiceFactory = ({{ requestId: approvalRequest.id, status: DelegatedApprovalRequestStatus.PendingDispatch },tx)) as TDelegatedApprovalRequest;- await emitDelegatedApprovalDispatchRequested(eventEmitter, { delegatedRequestId: request.id }, tx);+ await emitDelegatedApprovalDispatchRequested(+ eventEmitter,+ { delegatedRequestId: request.id, updatedAt: request.updatedAt },+ tx+ );return request;} catch (err) {if (hasPostgresErrorCode(err, DatabaseErrorCode.UniqueViolation)) {@@ -212,7 +216,10 @@ export const delegatedApprovalServiceFactory = ({});}- await emitDelegatedApprovalDispatchRequested(eventEmitter, { delegatedRequestId: delegatedRequest.id });+ await emitDelegatedApprovalDispatchRequested(eventEmitter, {+ delegatedRequestId: delegatedRequest.id,+ updatedAt: delegatedRequest.updatedAt+ });return { delegatedRequest, delegatedPolicy };};diff --git a/backend/src/services/delegated-approval/dispatch/delegated-approval-dispatch-consumer.test.ts b/backend/src/services/delegated-approval/dispatch/delegated-approval-dispatch-consumer.test.tsindex 0018143764..d334a5e08e 100644--- a/backend/src/services/delegated-approval/dispatch/delegated-approval-dispatch-consumer.test.ts+++ b/backend/src/services/delegated-approval/dispatch/delegated-approval-dispatch-consumer.test.ts@@ -215,6 +215,82 @@ describe("delegatedApprovalDispatchConsumer", () => {expect(dispatch).not.toHaveBeenCalled();});+ test("dispatches different requests together", async () => {+ const otherRequestId = "33333333-3333-3333-3333-333333333333";+ let releaseFirst: () => void = () => {};+ const firstHeld = new Promise<void>((resolve) => {+ releaseFirst = resolve;+ });++ dispatch.mockImplementation(async (input: { delegatedRequestId: string }) => {+ if (input.delegatedRequestId === DELEGATED_REQUEST_ID) await firstHeld;+ return { providerRequestId: `snow-${input.delegatedRequestId}` };+ });++ const { consumer } = buildConsumer({+ findById: async (id: string) => ({+ ...pendingDelegatedRequest,+ id,+ requestId: id === DELEGATED_REQUEST_ID ? "req-1" : "req-2"+ })+ });++ const pending = consumer.handle([+ makeEvent({ id: 1 }),+ makeEvent({ id: 2, payload: { delegatedRequestId: otherRequestId } })+ ]);++ try {+ await vi.waitFor(() => expect(dispatch).toHaveBeenCalledTimes(2));+ } finally {+ releaseFirst();+ }++ await expect(pending).resolves.toEqual([+ { id: "1", status: EventResultStatus.Delivered },+ { id: "2", status: EventResultStatus.Delivered }+ ]);+ });++ test("dispatches a second event for the same request only after the first settles", async () => {+ let releaseFirst: () => void = () => {};+ const firstHeld = new Promise<void>((resolve) => {+ releaseFirst = resolve;+ });+ let firstSettledAt = 0;+ let secondStartedAt = 0;++ dispatch.mockImplementation(async () => {+ if (dispatch.mock.calls.length === 1) {+ await firstHeld;+ firstSettledAt = Date.now();+ throw new Error("ServiceNow did not respond.");+ }+ secondStartedAt = Date.now();+ return { providerRequestId: "snow-2" };+ });++ const { consumer } = buildConsumer();+ const pending = consumer.handle([makeEvent({ id: 1 }), makeEvent({ id: 2 })]);++ try {+ await vi.waitFor(() => expect(dispatch).toHaveBeenCalledTimes(1));+ await new Promise((resolve) => {+ setTimeout(resolve, 30);+ });+ expect(dispatch).toHaveBeenCalledTimes(1);+ } finally {+ releaseFirst();+ }++ const results = await pending;++ expect(dispatch).toHaveBeenCalledTimes(2);+ expect(firstSettledAt).toBeGreaterThan(0);+ expect(secondStartedAt).toBeGreaterThanOrEqual(firstSettledAt);+ expect(results.map((result) => result.status)).toEqual([EventResultStatus.Retry, EventResultStatus.Delivered]);+ });+test("fails an unreadable payload terminally", async () => {const { consumer } = buildConsumer();diff --git a/backend/src/services/delegated-approval/dispatch/delegated-approval-dispatch-consumer.ts b/backend/src/services/delegated-approval/dispatch/delegated-approval-dispatch-consumer.tsindex 311ab65386..8322a0c325 100644--- a/backend/src/services/delegated-approval/dispatch/delegated-approval-dispatch-consumer.ts+++ b/backend/src/services/delegated-approval/dispatch/delegated-approval-dispatch-consumer.ts@@ -1,4 +1,5 @@import { UnrecoverableError } from "bullmq";+import pLimit from "p-limit";import { ProjectType } from "@app/db/schemas";import { logger } from "@app/lib/logger";@@ -35,6 +36,8 @@ import { TDelegatedApprovalDispatchInput } from "./delegated-approval-dispatch-texport const DELEGATED_APPROVAL_DISPATCH_CONSUMER = "delegated-approval-dispatch";+const DELEGATED_APPROVAL_DISPATCH_CONCURRENCY = 10;+type TDelegatedApprovalDispatchConsumerDep = {delegatedApprovalRequestDAL: Pick<TDelegatedApprovalRequestDALFactory,@@ -256,23 +259,55 @@ export const delegatedApprovalDispatchConsumerFactory = ({return { id, ...(await $dispatch(payload.data.delegatedRequestId)) };};- const handle = async (events: TEvent[]): Promise<TEventConsumerResult[]> => {- const results: TEventConsumerResult[] = [];-- for (const event of events) {- try {- // eslint-disable-next-line no-await-in-loop -- one dispatch at a time, so two events for the same request never race- results.push(await $handleEvent(event));- } catch (error) {- results.push({- id: String(event.id),- status: EventResultStatus.Retry,- error: error instanceof Error ? error.message : "Unknown error"- });- }+ const $runEvent = async (event: TEvent): Promise<TEventConsumerResult> => {+ try {+ return await $handleEvent(event);+ } catch (error) {+ return {+ id: String(event.id),+ status: EventResultStatus.Retry,+ error: error instanceof Error ? error.message : "Unknown error"+ };}+ };++ // Unreadable payloads have no request id, so each one is its own group and cannot block another request.+ const $groupByRequest = (events: TEvent[]): TEvent[][] => {+ const groups: TEvent[][] = [];+ const indexByRequest = new Map<string, number>();++ events.forEach((event) => {+ const payload = DelegatedApprovalDispatchRequestedPayloadSchema.safeParse(event.payload);+ const key = payload.success ? payload.data.delegatedRequestId : `event:${String(event.id)}`;+ const existing = indexByRequest.get(key);+ if (existing === undefined) {+ indexByRequest.set(key, groups.length);+ groups.push([event]);+ } else {+ groups[existing].push(event);+ }+ });- return results;+ return groups;+ };++ const handle = async (events: TEvent[]): Promise<TEventConsumerResult[]> => {+ const limit = pLimit(DELEGATED_APPROVAL_DISPATCH_CONCURRENCY);++ const groupResults = await Promise.all(+ $groupByRequest(events).map((group) =>+ limit(async () => {+ const results: TEventConsumerResult[] = [];+ for (const event of group) {+ // eslint-disable-next-line no-await-in-loop -- two events for one request must not both pass the providerRequestId check+ results.push(await $runEvent(event));+ }+ return results;+ })+ )+ );++ return groupResults.flat();};return {