Skip to content

Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.

add plimit and insert updatedAt on the idepotency key to avoid retry burst at the same time

adilsonc98a55ad7859
diff --git a/CLAUDE.md b/CLAUDE.md
index 4f7b46bc22..a74c2aa08d 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -169,8 +169,10 @@ failure returns `Retry`, so the outbox backs off and tries again (up to `MAX_OUT
promises that retry on every attempt because the consumer cannot see the attempt count. A user resends a
`failed_dispatch` request with `POST /api/v1/access-approvals/requests/:requestId/retry-delegated-dispatch`. The
product's bridge checks who may retry, and `delegatedApprovalService.retryDispatch` checks the request can be
-retried and emits the same event with no transaction. One flush runs per consumer at a time, so two dispatches
-of one request never run together; a second event for a request that was already delivered is marked delivered
+retried and emits the same event with no transaction. The emit's idempotency key is the event, the delegated
+request id, and that row's `updatedAt`, so repeated emits of one attempt collapse, and a later resend still
+inserts after the status write bumps `updatedAt`. A batch dispatches up to 10 different requests at once;
+events for one request stay serial. A second event for a request that was already delivered is marked delivered
without calling the provider. A delegated request has
no Infisical approvers, so `POST /api/v1/access-approvals/requests/:requestId/cancel-delegated-dispatch` is the only way to close
one from Infisical: the requester, or anyone who can edit the policy, sets it `cancelled`, which is also how a
diff --git a/backend/src/services/delegated-approval/delegated-approval-events.ts b/backend/src/services/delegated-approval/delegated-approval-events.ts
index be728eea7d..0e9edc777d 100644
--- a/backend/src/services/delegated-approval/delegated-approval-events.ts
+++ b/backend/src/services/delegated-approval/delegated-approval-events.ts
@@ -13,16 +13,19 @@ export type TDelegatedApprovalDispatchRequestedPayload = z.infer<
typeof DelegatedApprovalDispatchRequestedPayloadSchema
>;
-// No idempotency key on purpose: a user retry has to insert a fresh row while the delivered one from the
-// first attempt is still retained, and the consumer already skips a request that was delivered.
+// updatedAt is the attempt. Repeated retries of one failure share it and collapse on the outbox unique
+// key. A request id alone would not: the previous row is still retained (delivered for a day, failed for
+// 30 days), so a later resend has to insert beside it. The consumer's status write bumps updatedAt,
+// which is what gives that resend a new key.
export const emitDelegatedApprovalDispatchRequested = (
eventEmitter: TEventEmitter,
- { delegatedRequestId }: TDelegatedApprovalDispatchRequestedPayload,
+ { delegatedRequestId, updatedAt }: TDelegatedApprovalDispatchRequestedPayload & { updatedAt: Date },
tx?: Knex
): Promise<void> =>
eventEmitter.emit(
{
eventType: DELEGATED_APPROVAL_DISPATCH_REQUESTED_EVENT,
+ idempotencyKey: `${DELEGATED_APPROVAL_DISPATCH_REQUESTED_EVENT}:${delegatedRequestId}:${updatedAt.toISOString()}`,
payload: { delegatedRequestId }
},
tx
diff --git a/backend/src/services/delegated-approval/delegated-approval-service.test.ts b/backend/src/services/delegated-approval/delegated-approval-service.test.ts
index 8f3118b4ab..1fc4369419 100644
--- a/backend/src/services/delegated-approval/delegated-approval-service.test.ts
+++ b/backend/src/services/delegated-approval/delegated-approval-service.test.ts
@@ -7,7 +7,13 @@ import { DelegatedApprovalPolicyType, DelegatedApprovalRequestStatus } from "./d
import { DELEGATED_APPROVAL_DISPATCH_REQUESTED_EVENT } from "./delegated-approval-events";
import { delegatedApprovalServiceFactory } from "./delegated-approval-service";
-const failedRequest = { id: "delegated-1", requestId: "req-1", status: DelegatedApprovalRequestStatus.FailedDispatch };
+const UPDATED_AT = new Date("2026-10-23T13:00:00.000Z");
+const failedRequest = {
+ id: "delegated-1",
+ requestId: "req-1",
+ status: DelegatedApprovalRequestStatus.FailedDispatch,
+ updatedAt: UPDATED_AT
+};
const serviceNowPolicy = {
id: "delegated-policy-1",
policyId: "policy-1",
@@ -31,6 +37,7 @@ const createService = ({
const emit = vi.fn().mockResolvedValue(undefined);
const create = vi.fn().mockImplementation(async (row: { requestId: string; status: string }) => ({
id: "delegated-1",
+ updatedAt: UPDATED_AT,
...row
}));
const service = delegatedApprovalServiceFactory({
@@ -43,6 +50,7 @@ const createService = ({
const dispatchRequestedEvent = {
eventType: DELEGATED_APPROVAL_DISPATCH_REQUESTED_EVENT,
+ idempotencyKey: `${DELEGATED_APPROVAL_DISPATCH_REQUESTED_EVENT}:delegated-1:${UPDATED_AT.toISOString()}`,
payload: { delegatedRequestId: "delegated-1" }
};
@@ -73,6 +81,17 @@ describe("delegatedApprovalService.retryDispatch", () => {
expect(emit).toHaveBeenCalledWith(dispatchRequestedEvent, undefined);
});
+ test("emits the same idempotency key when the same failed attempt is sent again", async () => {
+ const { service, emit } = createService();
+
+ await service.retryDispatch({ approvalRequest: pendingApprovalRequest });
+ await service.retryDispatch({ approvalRequest: pendingApprovalRequest });
+
+ expect(emit).toHaveBeenCalledTimes(2);
+ expect(emit).toHaveBeenNthCalledWith(1, dispatchRequestedEvent, undefined);
+ expect(emit).toHaveBeenNthCalledWith(2, dispatchRequestedEvent, undefined);
+ });
+
test("refuses a request without a delegated approval", async () => {
const { service, emit } = createService({ delegatedRequest: null });
diff --git a/backend/src/services/delegated-approval/delegated-approval-service.ts b/backend/src/services/delegated-approval/delegated-approval-service.ts
index e2fe2f857a..0be7efbd01 100644
--- a/backend/src/services/delegated-approval/delegated-approval-service.ts
+++ b/backend/src/services/delegated-approval/delegated-approval-service.ts
@@ -157,7 +157,11 @@ export const delegatedApprovalServiceFactory = ({
{ requestId: approvalRequest.id, status: DelegatedApprovalRequestStatus.PendingDispatch },
tx
)) as TDelegatedApprovalRequest;
- await emitDelegatedApprovalDispatchRequested(eventEmitter, { delegatedRequestId: request.id }, tx);
+ await emitDelegatedApprovalDispatchRequested(
+ eventEmitter,
+ { delegatedRequestId: request.id, updatedAt: request.updatedAt },
+ tx
+ );
return request;
} catch (err) {
if (hasPostgresErrorCode(err, DatabaseErrorCode.UniqueViolation)) {
@@ -212,7 +216,10 @@ export const delegatedApprovalServiceFactory = ({
});
}
- await emitDelegatedApprovalDispatchRequested(eventEmitter, { delegatedRequestId: delegatedRequest.id });
+ await emitDelegatedApprovalDispatchRequested(eventEmitter, {
+ delegatedRequestId: delegatedRequest.id,
+ updatedAt: delegatedRequest.updatedAt
+ });
return { delegatedRequest, delegatedPolicy };
};
diff --git a/backend/src/services/delegated-approval/dispatch/delegated-approval-dispatch-consumer.test.ts b/backend/src/services/delegated-approval/dispatch/delegated-approval-dispatch-consumer.test.ts
index 0018143764..d334a5e08e 100644
--- a/backend/src/services/delegated-approval/dispatch/delegated-approval-dispatch-consumer.test.ts
+++ b/backend/src/services/delegated-approval/dispatch/delegated-approval-dispatch-consumer.test.ts
@@ -215,6 +215,82 @@ describe("delegatedApprovalDispatchConsumer", () => {
expect(dispatch).not.toHaveBeenCalled();
});
+ test("dispatches different requests together", async () => {
+ const otherRequestId = "33333333-3333-3333-3333-333333333333";
+ let releaseFirst: () => void = () => {};
+ const firstHeld = new Promise<void>((resolve) => {
+ releaseFirst = resolve;
+ });
+
+ dispatch.mockImplementation(async (input: { delegatedRequestId: string }) => {
+ if (input.delegatedRequestId === DELEGATED_REQUEST_ID) await firstHeld;
+ return { providerRequestId: `snow-${input.delegatedRequestId}` };
+ });
+
+ const { consumer } = buildConsumer({
+ findById: async (id: string) => ({
+ ...pendingDelegatedRequest,
+ id,
+ requestId: id === DELEGATED_REQUEST_ID ? "req-1" : "req-2"
+ })
+ });
+
+ const pending = consumer.handle([
+ makeEvent({ id: 1 }),
+ makeEvent({ id: 2, payload: { delegatedRequestId: otherRequestId } })
+ ]);
+
+ try {
+ await vi.waitFor(() => expect(dispatch).toHaveBeenCalledTimes(2));
+ } finally {
+ releaseFirst();
+ }
+
+ await expect(pending).resolves.toEqual([
+ { id: "1", status: EventResultStatus.Delivered },
+ { id: "2", status: EventResultStatus.Delivered }
+ ]);
+ });
+
+ test("dispatches a second event for the same request only after the first settles", async () => {
+ let releaseFirst: () => void = () => {};
+ const firstHeld = new Promise<void>((resolve) => {
+ releaseFirst = resolve;
+ });
+ let firstSettledAt = 0;
+ let secondStartedAt = 0;
+
+ dispatch.mockImplementation(async () => {
+ if (dispatch.mock.calls.length === 1) {
+ await firstHeld;
+ firstSettledAt = Date.now();
+ throw new Error("ServiceNow did not respond.");
+ }
+ secondStartedAt = Date.now();
+ return { providerRequestId: "snow-2" };
+ });
+
+ const { consumer } = buildConsumer();
+ const pending = consumer.handle([makeEvent({ id: 1 }), makeEvent({ id: 2 })]);
+
+ try {
+ await vi.waitFor(() => expect(dispatch).toHaveBeenCalledTimes(1));
+ await new Promise((resolve) => {
+ setTimeout(resolve, 30);
+ });
+ expect(dispatch).toHaveBeenCalledTimes(1);
+ } finally {
+ releaseFirst();
+ }
+
+ const results = await pending;
+
+ expect(dispatch).toHaveBeenCalledTimes(2);
+ expect(firstSettledAt).toBeGreaterThan(0);
+ expect(secondStartedAt).toBeGreaterThanOrEqual(firstSettledAt);
+ expect(results.map((result) => result.status)).toEqual([EventResultStatus.Retry, EventResultStatus.Delivered]);
+ });
+
test("fails an unreadable payload terminally", async () => {
const { consumer } = buildConsumer();
diff --git a/backend/src/services/delegated-approval/dispatch/delegated-approval-dispatch-consumer.ts b/backend/src/services/delegated-approval/dispatch/delegated-approval-dispatch-consumer.ts
index 311ab65386..8322a0c325 100644
--- a/backend/src/services/delegated-approval/dispatch/delegated-approval-dispatch-consumer.ts
+++ b/backend/src/services/delegated-approval/dispatch/delegated-approval-dispatch-consumer.ts
@@ -1,4 +1,5 @@
import { UnrecoverableError } from "bullmq";
+import pLimit from "p-limit";
import { ProjectType } from "@app/db/schemas";
import { logger } from "@app/lib/logger";
@@ -35,6 +36,8 @@ import { TDelegatedApprovalDispatchInput } from "./delegated-approval-dispatch-t
export const DELEGATED_APPROVAL_DISPATCH_CONSUMER = "delegated-approval-dispatch";
+const DELEGATED_APPROVAL_DISPATCH_CONCURRENCY = 10;
+
type TDelegatedApprovalDispatchConsumerDep = {
delegatedApprovalRequestDAL: Pick<
TDelegatedApprovalRequestDALFactory,
@@ -256,23 +259,55 @@ export const delegatedApprovalDispatchConsumerFactory = ({
return { id, ...(await $dispatch(payload.data.delegatedRequestId)) };
};
- const handle = async (events: TEvent[]): Promise<TEventConsumerResult[]> => {
- const results: TEventConsumerResult[] = [];
-
- for (const event of events) {
- try {
- // eslint-disable-next-line no-await-in-loop -- one dispatch at a time, so two events for the same request never race
- results.push(await $handleEvent(event));
- } catch (error) {
- results.push({
- id: String(event.id),
- status: EventResultStatus.Retry,
- error: error instanceof Error ? error.message : "Unknown error"
- });
- }
+ const $runEvent = async (event: TEvent): Promise<TEventConsumerResult> => {
+ try {
+ return await $handleEvent(event);
+ } catch (error) {
+ return {
+ id: String(event.id),
+ status: EventResultStatus.Retry,
+ error: error instanceof Error ? error.message : "Unknown error"
+ };
}
+ };
+
+ // Unreadable payloads have no request id, so each one is its own group and cannot block another request.
+ const $groupByRequest = (events: TEvent[]): TEvent[][] => {
+ const groups: TEvent[][] = [];
+ const indexByRequest = new Map<string, number>();
+
+ events.forEach((event) => {
+ const payload = DelegatedApprovalDispatchRequestedPayloadSchema.safeParse(event.payload);
+ const key = payload.success ? payload.data.delegatedRequestId : `event:${String(event.id)}`;
+ const existing = indexByRequest.get(key);
+ if (existing === undefined) {
+ indexByRequest.set(key, groups.length);
+ groups.push([event]);
+ } else {
+ groups[existing].push(event);
+ }
+ });
- return results;
+ return groups;
+ };
+
+ const handle = async (events: TEvent[]): Promise<TEventConsumerResult[]> => {
+ const limit = pLimit(DELEGATED_APPROVAL_DISPATCH_CONCURRENCY);
+
+ const groupResults = await Promise.all(
+ $groupByRequest(events).map((group) =>
+ limit(async () => {
+ const results: TEventConsumerResult[] = [];
+ for (const event of group) {
+ // eslint-disable-next-line no-await-in-loop -- two events for one request must not both pass the providerRequestId check
+ results.push(await $runEvent(event));
+ }
+ return results;
+ })
+ )
+ );
+
+ return groupResults.flat();
};
return {