Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.
TypeScript acmeagentsai-agentscertificate-managementdevsecopsenvironment-variableskubernetesopen-sourcepampkipostgresprivate-caprivileged-access-managementsecret-managementsecret-managersecretssecrets-managementsecuritysecurity-toolsvault
supersed the old event so we don't have two concurrent events trying to retry
adilsondd75b38e1b57
diff --git a/AGENTS.md b/AGENTS.mdindex c40b94d8d0..e19e21c08d 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -183,9 +183,11 @@ request goes `cancelled`, and the consumer returns `Failed`. A terminal failure`failed_dispatch` on every attempt, because an admin can fix the cause and resend. A user resends a`failed_dispatch` request with `POST /api/v1/access-approvals/requests/:requestId/retry-delegated-dispatch`. Theproduct's bridge checks who may retry, and `delegatedApprovalService.retryDispatch` checks the request can be-retried and emits the same event with no transaction. The event id is derived (uuid v5) from the delegated-request id and that row's `updatedAt`, so repeated emits of one attempt collapse on the outbox's-`(consumer, eventId)` key, and a later resend still inserts after the status write bumps `updatedAt`. A batch dispatches up to 10 different requests at once;+retried. A resend supersedes the event still in the outbox rather than running beside it: the row stores the+live event id in `dispatchEventId`, and a resend writes a new random one (only while the row is+`failed_dispatch`) and emits the event under it in one transaction. The consumer marks any event whose id no+longer matches delivered without calling the provider or writing the row, so an older event's last attempt+can never close a request the user resent, and concurrent resends call the provider once. A batch dispatches up to 10 different requests at once;events for one request stay serial. A second event for a request that was already delivered is marked deliveredwithout calling the provider. A delegated request hasno Infisical approvers, so `POST /api/v1/access-approvals/requests/:requestId/cancel-delegated-dispatch` is the only way to closediff --git a/backend/src/db/migrations/20261028120000_delegated-approval-request-dispatch-event-id.ts b/backend/src/db/migrations/20261028120000_delegated-approval-request-dispatch-event-id.tsnew file mode 100644index 0000000000..e7cecfafe5--- /dev/null+++ b/backend/src/db/migrations/20261028120000_delegated-approval-request-dispatch-event-id.ts@@ -0,0 +1,24 @@+import { Knex } from "knex";++import { TableName } from "../schemas";++export async function up(knex: Knex): Promise<void> {+ const hasColumn = await knex.schema.hasColumn(TableName.DelegatedApprovalRequests, "dispatchEventId");+ if (!hasColumn) {+ await knex.schema.alterTable(TableName.DelegatedApprovalRequests, (t) => {+ t.uuid("dispatchEventId").notNullable().defaultTo(knex.fn.uuid());+ });+ // The default only backfills existing rows. The id must be the one the caller emits its event under, so+ // a create that omits it should fail rather than get an id no event carries.+ await knex.raw('ALTER TABLE ?? ALTER COLUMN "dispatchEventId" DROP DEFAULT', [TableName.DelegatedApprovalRequests]);+ }+}++export async function down(knex: Knex): Promise<void> {+ const hasColumn = await knex.schema.hasColumn(TableName.DelegatedApprovalRequests, "dispatchEventId");+ if (hasColumn) {+ await knex.schema.alterTable(TableName.DelegatedApprovalRequests, (t) => {+ t.dropColumn("dispatchEventId");+ });+ }+}diff --git a/backend/src/db/schemas/delegated-approval-requests.ts b/backend/src/db/schemas/delegated-approval-requests.tsindex 950f8c0f7e..91d880de98 100644--- a/backend/src/db/schemas/delegated-approval-requests.ts+++ b/backend/src/db/schemas/delegated-approval-requests.ts@@ -17,7 +17,8 @@ export const DelegatedApprovalRequestsSchema = z.object({statusReason: z.string().nullable().optional(),providerRequestId: z.string().nullable().optional(),createdAt: z.date(),- updatedAt: z.date()+ updatedAt: z.date(),+ dispatchEventId: z.string().uuid()});export type TDelegatedApprovalRequests = z.infer<typeof DelegatedApprovalRequestsSchema>;diff --git a/backend/src/services/delegated-approval/delegated-approval-events.ts b/backend/src/services/delegated-approval/delegated-approval-events.tsindex 3c346512d2..5a38abc28a 100644--- a/backend/src/services/delegated-approval/delegated-approval-events.ts+++ b/backend/src/services/delegated-approval/delegated-approval-events.ts@@ -1,14 +1,10 @@import { Knex } from "knex";-import { v5 as uuidv5 } from "uuid";import { z } from "zod";import { TEventEmitter } from "@app/services/event-outbox/event-outbox-types";export const DELEGATED_APPROVAL_DISPATCH_REQUESTED_EVENT = "delegated-approval.request.dispatch-requested";-// Namespaces the event ids derived below, so they can't collide with ids derived the same way elsewhere.-const DELEGATED_APPROVAL_DISPATCH_REQUESTED_EVENT_ID_NAMESPACE = "95183bf5-9016-4f76-9263-b7682f7e6a14";-export const DelegatedApprovalDispatchRequestedPayloadSchema = z.object({delegatedRequestId: z.string().uuid()});@@ -17,25 +13,17 @@ export type TDelegatedApprovalDispatchRequestedPayload = z.infer<typeof DelegatedApprovalDispatchRequestedPayloadSchema>;-// The id is derived from updatedAt, which is the attempt. Repeated retries of one failure share it and-// the outbox drops all but the first. A request id alone would not do: the previous row is still retained-// (delivered for a day, failed for 30 days), so a later resend has to insert beside it. The consumer's-// status write bumps updatedAt, which is what gives that resend a new id.-export const buildDelegatedApprovalDispatchRequestedEventId = ({- delegatedRequestId,- updatedAt-}: TDelegatedApprovalDispatchRequestedPayload & { updatedAt: Date }): string =>- uuidv5(`${delegatedRequestId}:${updatedAt.toISOString()}`, DELEGATED_APPROVAL_DISPATCH_REQUESTED_EVENT_ID_NAMESPACE);-+// The event id is the request's dispatchEventId. A resend stores a new one, and the consumer skips any event+// whose id no longer matches, so an older event still in backoff can't dispatch or close the request.export const emitDelegatedApprovalDispatchRequested = (eventEmitter: TEventEmitter,- { delegatedRequestId, updatedAt }: TDelegatedApprovalDispatchRequestedPayload & { updatedAt: Date },+ { delegatedRequestId, dispatchEventId }: TDelegatedApprovalDispatchRequestedPayload & { dispatchEventId: string },tx?: Knex): Promise<void> =>eventEmitter.emit([{- id: buildDelegatedApprovalDispatchRequestedEventId({ delegatedRequestId, updatedAt }),+ id: dispatchEventId,type: DELEGATED_APPROVAL_DISPATCH_REQUESTED_EVENT,data: { delegatedRequestId }}diff --git a/backend/src/services/delegated-approval/delegated-approval-service.test.ts b/backend/src/services/delegated-approval/delegated-approval-service.test.tsindex b4cf047ef1..2987a7128f 100644--- a/backend/src/services/delegated-approval/delegated-approval-service.test.ts+++ b/backend/src/services/delegated-approval/delegated-approval-service.test.ts@@ -4,18 +4,14 @@ import { BadRequestError } from "@app/lib/errors";import { ApprovalPolicyType, ApprovalRequestStatus } from "@app/services/approval-policy/approval-policy-enums";import { DelegatedApprovalPolicyType, DelegatedApprovalRequestStatus } from "./delegated-approval-enums";-import {- buildDelegatedApprovalDispatchRequestedEventId,- DELEGATED_APPROVAL_DISPATCH_REQUESTED_EVENT-} from "./delegated-approval-events";+import { DELEGATED_APPROVAL_DISPATCH_REQUESTED_EVENT } from "./delegated-approval-events";import { delegatedApprovalServiceFactory } from "./delegated-approval-service";-const UPDATED_AT = new Date("2026-10-23T13:00:00.000Z");const failedRequest = {id: "delegated-1",requestId: "req-1",status: DelegatedApprovalRequestStatus.FailedDispatch,- updatedAt: UPDATED_AT+ dispatchEventId: "44444444-4444-4444-8444-444444444444"};const serviceNowPolicy = {id: "delegated-policy-1",@@ -36,26 +32,37 @@ const PRIMARY = { primary: true } as never;const createService = ({delegatedRequest = failedRequest as unknown,- delegatedPolicy = serviceNowPolicy as unknown+ delegatedPolicy = serviceNowPolicy as unknown,+ resendMatches = true} = {}) => {const emit = vi.fn().mockResolvedValue(undefined);const findOne = vi.fn().mockResolvedValue(delegatedRequest);const create = vi.fn().mockImplementation(async (row: { requestId: string; status: string }) => ({id: "delegated-1",- updatedAt: UPDATED_AT,...row}));+ const update = vi+ .fn()+ .mockImplementation(async (_filter: unknown, patch: { dispatchEventId: string }) =>+ resendMatches ? [{ ...(delegatedRequest as object), ...patch }] : []+ );const service = delegatedApprovalServiceFactory({- delegatedApprovalRequestDAL: { findOne, create, primaryNode: () => PRIMARY },+ delegatedApprovalRequestDAL: {+ findOne,+ create,+ update,+ primaryNode: () => PRIMARY,+ transaction: async (cb: (tx: never) => Promise<unknown>) => cb(TX)+ },delegatedApprovalPolicyDAL: { findOne: vi.fn().mockResolvedValue(delegatedPolicy) },eventEmitter: { emit }} as never);- return { service, emit, create, findOne };+ return { service, emit, create, update, findOne };};-const dispatchRequestedEvent = [+const dispatchRequestedEvent = (id: string) => [{- id: buildDelegatedApprovalDispatchRequestedEventId({ delegatedRequestId: "delegated-1", updatedAt: UPDATED_AT }),+ id,type: DELEGATED_APPROVAL_DISPATCH_REQUESTED_EVENT,data: { delegatedRequestId: "delegated-1" }}@@ -69,35 +76,55 @@ describe("delegatedApprovalService.createRequest", () => {expect(request).toMatchObject({ id: "delegated-1", status: DelegatedApprovalRequestStatus.PendingDispatch });expect(create).toHaveBeenCalledWith(- { requestId: "req-1", status: DelegatedApprovalRequestStatus.PendingDispatch },+ {+ requestId: "req-1",+ status: DelegatedApprovalRequestStatus.PendingDispatch,+ dispatchEventId: expect.any(String) as string+ },TX);expect(emit).toHaveBeenCalledTimes(1);- expect(emit).toHaveBeenCalledWith(dispatchRequestedEvent, TX);+ expect(emit).toHaveBeenCalledWith(dispatchRequestedEvent(request.dispatchEventId), TX);});});describe("delegatedApprovalService.retryDispatch", () => {- test("asks for a new dispatch of a pending request that failed to reach the delegated system", async () => {- const { service, emit, findOne } = createService();+ test("supersedes the previous dispatch event with a new one in the same transaction", async () => {+ const { service, emit, update, findOne } = createService();- await expect(service.retryDispatch({ approvalRequest: pendingApprovalRequest })).resolves.toEqual({- delegatedRequest: failedRequest,- delegatedPolicy: serviceNowPolicy+ const { delegatedRequest, delegatedPolicy } = await service.retryDispatch({+ approvalRequest: pendingApprovalRequest});++ expect(delegatedPolicy).toEqual(serviceNowPolicy);+ expect(delegatedRequest.dispatchEventId).not.toBe(failedRequest.dispatchEventId);expect(findOne).toHaveBeenCalledWith({ requestId: "req-1" }, PRIMARY);- expect(emit).toHaveBeenCalledWith(dispatchRequestedEvent, undefined);+ expect(update).toHaveBeenCalledWith(+ { id: "delegated-1", status: DelegatedApprovalRequestStatus.FailedDispatch },+ { dispatchEventId: delegatedRequest.dispatchEventId },+ TX+ );+ expect(emit).toHaveBeenCalledWith(dispatchRequestedEvent(delegatedRequest.dispatchEventId), TX);});- test("emits the same event id when the same failed attempt is sent again", async () => {+ test("emits a new event id on every resend", async () => {const { service, emit } = createService();await service.retryDispatch({ approvalRequest: pendingApprovalRequest });await service.retryDispatch({ approvalRequest: pendingApprovalRequest });expect(emit).toHaveBeenCalledTimes(2);- expect(emit).toHaveBeenNthCalledWith(1, dispatchRequestedEvent, undefined);- expect(emit).toHaveBeenNthCalledWith(2, dispatchRequestedEvent, undefined);+ const [[[first]], [[second]]] = emit.mock.calls as [{ id: string }[]][];+ expect(first.id).not.toBe(second.id);+ });++ test("refuses a resend when the request left failed_dispatch before it landed", async () => {+ const { service, emit } = createService({ resendMatches: false });++ await expect(service.retryDispatch({ approvalRequest: pendingApprovalRequest })).rejects.toThrow(+ "Only requests that failed to reach ServiceNow can be sent again"+ );+ expect(emit).not.toHaveBeenCalled();});test("refuses a request without a delegated approval", async () => {@@ -145,11 +172,12 @@ describe("delegatedApprovalService.retryDispatch", () => {DelegatedApprovalRequestStatus.WaitingApproval,DelegatedApprovalRequestStatus.Approved])("refuses a request in the %s state", async (status) => {- const { service, emit } = createService({ delegatedRequest: { ...failedRequest, status } });+ const { service, emit, update } = createService({ delegatedRequest: { ...failedRequest, status } });await expect(service.retryDispatch({ approvalRequest: pendingApprovalRequest })).rejects.toThrow("Only requests that failed to reach ServiceNow can be sent again");+ expect(update).not.toHaveBeenCalled();expect(emit).not.toHaveBeenCalled();});});diff --git a/backend/src/services/delegated-approval/delegated-approval-service.ts b/backend/src/services/delegated-approval/delegated-approval-service.tsindex 063f42988f..5f752d6949 100644--- a/backend/src/services/delegated-approval/delegated-approval-service.ts+++ b/backend/src/services/delegated-approval/delegated-approval-service.ts@@ -1,3 +1,5 @@+import crypto from "node:crypto";+import { Knex } from "knex";import { DatabaseErrorCode } from "@app/lib/error-codes";@@ -152,14 +154,15 @@ export const delegatedApprovalServiceFactory = ({const createRequest = async ({ approvalRequest }: TCreateDelegatedApprovalRequestDTO, tx?: Knex) => {assertPolicyTypeSupportsDelegatedApproval(approvalRequest.type);+ const dispatchEventId = crypto.randomUUID();try {const request = (await delegatedApprovalRequestDAL.create(- { requestId: approvalRequest.id, status: DelegatedApprovalRequestStatus.PendingDispatch },+ { requestId: approvalRequest.id, status: DelegatedApprovalRequestStatus.PendingDispatch, dispatchEventId },tx)) as TDelegatedApprovalRequest;await emitDelegatedApprovalDispatchRequested(eventEmitter,- { delegatedRequestId: request.id, updatedAt: request.updatedAt },+ { delegatedRequestId: request.id, dispatchEventId },tx);return request;@@ -210,18 +213,34 @@ export const delegatedApprovalServiceFactory = ({});}+ const notResendableError = new BadRequestError({+ message: `Only requests that failed to reach ${DELEGATED_APPROVAL_NAME_MAP[delegatedPolicy.type]} can be sent again`+ });+if (delegatedRequest.status !== DelegatedApprovalRequestStatus.FailedDispatch) {- throw new BadRequestError({- message: `Only requests that failed to reach ${DELEGATED_APPROVAL_NAME_MAP[delegatedPolicy.type]} can be sent again`- });+ throw notResendableError;}- await emitDelegatedApprovalDispatchRequested(eventEmitter, {- delegatedRequestId: delegatedRequest.id,- updatedAt: delegatedRequest.updatedAt+ // Conditional on failed_dispatch so it serializes with the consumer reopening the row for the event this+ // replaces: whichever takes the row first wins, and the other backs off.+ const dispatchEventId = crypto.randomUUID();+ const resentRequest = await delegatedApprovalRequestDAL.transaction(async (tx) => {+ const [resent] = await delegatedApprovalRequestDAL.update(+ { id: delegatedRequest.id, status: DelegatedApprovalRequestStatus.FailedDispatch },+ { dispatchEventId },+ tx+ );+ if (!resent) throw notResendableError;++ await emitDelegatedApprovalDispatchRequested(+ eventEmitter,+ { delegatedRequestId: resent.id, dispatchEventId },+ tx+ );+ return resent as TDelegatedApprovalRequest;});- return { delegatedRequest, delegatedPolicy };+ return { delegatedRequest: resentRequest, delegatedPolicy };};const updateRequest = async ({ requestId, status, statusReason }: TUpdateDelegatedApprovalRequestDTO, tx?: Knex) => {diff --git a/backend/src/services/delegated-approval/dispatch/delegated-approval-dispatch-consumer.test.ts b/backend/src/services/delegated-approval/dispatch/delegated-approval-dispatch-consumer.test.tsindex bd92e61c31..9304339089 100644--- a/backend/src/services/delegated-approval/dispatch/delegated-approval-dispatch-consumer.test.ts+++ b/backend/src/services/delegated-approval/dispatch/delegated-approval-dispatch-consumer.test.ts@@ -51,6 +51,7 @@ vi.mock("@app/services/app-connection/app-connection-fns", () => ({const DELEGATED_REQUEST_ID = "11111111-1111-1111-1111-111111111111";const EVENT_ID = "44444444-4444-4444-8444-444444444444";+const RESENT_EVENT_ID = "55555555-5555-4555-8555-555555555555";const ORG_ID = "22222222-2222-2222-2222-222222222222";const RETRY_REASON ="ServiceNow did not respond. Infisical will retry automatically. If it keeps failing, send the request again.";@@ -61,7 +62,8 @@ const pendingDelegatedRequest = {status: DelegatedApprovalRequestStatus.PendingDispatch,statusReason: null as string | null,providerRequestId: null as string | null,- encryptedCallbackToken: null+ encryptedCallbackToken: null,+ dispatchEventId: "event-1"};type TRow = typeof pendingDelegatedRequest;@@ -284,7 +286,8 @@ describe("delegatedApprovalDispatchConsumer", () => {findById: async (id: string) => ({...pendingDelegatedRequest,id,- requestId: id === DELEGATED_REQUEST_ID ? "req-1" : "req-2"+ requestId: id === DELEGATED_REQUEST_ID ? "req-1" : "req-2",+ dispatchEventId: id === DELEGATED_REQUEST_ID ? "event-1" : "event-2"})});@@ -305,45 +308,102 @@ describe("delegatedApprovalDispatchConsumer", () => {]);});- test("dispatches a second event for the same request only after the first settles", async () => {+ test("handles a second event for the same request only after the first settles", async () => {let releaseFirst: () => void = () => {};const firstHeld = new Promise<void>((resolve) => {releaseFirst = resolve;});- let firstSettledAt = 0;- let secondStartedAt = 0;dispatch.mockImplementation(async () => {- if (dispatch.mock.calls.length === 1) {- await firstHeld;- firstSettledAt = Date.now();- throw new Error("ServiceNow did not respond.");- }- secondStartedAt = Date.now();- return { providerRequestId: "snow-2" };+ await firstHeld;+ throw new Error("ServiceNow did not respond.");});- const { consumer } = buildConsumer();- const pending = consumer.handle([makeEvent({ id: "event-1" }), makeEvent({ id: "event-2" })]);+ let readRow: () => TRow | null = () => null;+ const findById = vi.fn(async () => readRow());+ const built = buildConsumer({ findById });+ readRow = built.row;++ const pending = built.consumer.handle([makeEvent({ id: "event-1" }), makeEvent({ id: "event-2" })]);try {await vi.waitFor(() => expect(dispatch).toHaveBeenCalledTimes(1));await new Promise((resolve) => {setTimeout(resolve, 30);});- expect(dispatch).toHaveBeenCalledTimes(1);+ expect(findById).toHaveBeenCalledTimes(1);} finally {releaseFirst();}const results = await pending;- expect(dispatch).toHaveBeenCalledTimes(2);- expect(firstSettledAt).toBeGreaterThan(0);- expect(secondStartedAt).toBeGreaterThanOrEqual(firstSettledAt);+ expect(findById).toHaveBeenCalledTimes(2);expect(results.map((result) => result.status)).toEqual([EventResultStatus.Retry, EventResultStatus.Delivered]);});+ test("skips a superseded event batched with the event that replaced it", async () => {+ dispatch.mockResolvedValueOnce({ providerRequestId: "snow-2" });+ const { consumer, row } = buildConsumer({+ delegatedRequest: { ...pendingDelegatedRequest, dispatchEventId: "event-2" }+ });++ const results = await consumer.handle([makeEvent({ id: "event-1" }), makeEvent({ id: "event-2" })]);++ expect(results).toEqual([+ { id: "event-1", status: EventResultStatus.Delivered },+ { id: "event-2", status: EventResultStatus.Delivered }+ ]);+ expect(dispatch).toHaveBeenCalledTimes(1);+ expect(row()).toMatchObject({+ status: DelegatedApprovalRequestStatus.WaitingApproval,+ providerRequestId: "snow-2"+ });+ });++ test.each([+ ["an earlier attempt", false],+ ["its last attempt", true]+ ])(+ "marks an event superseded by a resend delivered on %s without dispatching or closing the request",+ async (_, isFinalAttempt) => {+ const resent = {+ ...pendingDelegatedRequest,+ status: DelegatedApprovalRequestStatus.FailedDispatch,+ statusReason: RETRY_REASON,+ dispatchEventId: "event-2"+ };+ const { consumer, row, update, approvalUpdate } = buildConsumer({ delegatedRequest: resent });++ const results = await consumer.handle([makeEvent({ id: "event-1", isFinalAttempt })]);++ expect(results).toEqual([{ id: "event-1", status: EventResultStatus.Delivered }]);+ expect(dispatch).not.toHaveBeenCalled();+ expect(update).not.toHaveBeenCalled();+ expect(approvalUpdate).not.toHaveBeenCalled();+ expect(row()).toEqual(resent);+ }+ );++ test("stops when a resend lands between loading the request and reopening it", async () => {+ const resent = {+ ...pendingDelegatedRequest,+ status: DelegatedApprovalRequestStatus.FailedDispatch,+ statusReason: RETRY_REASON,+ dispatchEventId: "event-2"+ };+ const { consumer, row } = buildConsumer({+ delegatedRequest: resent,+ findById: async () => ({ ...resent, dispatchEventId: "event-1" })+ });++ const results = await consumer.handle([makeEvent({ id: "event-1" })]);++ expect(results).toEqual([{ id: "event-1", status: EventResultStatus.Delivered }]);+ expect(dispatch).not.toHaveBeenCalled();+ expect(row()).toEqual(resent);+ });+test("fails an unreadable payload terminally", async () => {const { consumer } = buildConsumer();@@ -358,6 +418,8 @@ describe("delegatedApprovalDispatchConsumer", () => {// Why this consumer exists: a dispatch that fails is kept by the outbox and tried again, instead of// depending on an enqueue that ran after the request committed and could be lost with it.describe("delegatedApprovalDispatchConsumer through the outbox", () => {+ const outboxDelegatedRequest = { ...pendingDelegatedRequest, dispatchEventId: EVENT_ID };+const buildOutbox = (consumer: ReturnType<typeof buildConsumer>["consumer"]) => {const registry = eventOutboxRegistryFactory();registry.register(consumer);@@ -429,7 +491,7 @@ describe("delegatedApprovalDispatchConsumer through the outbox", () => {dispatch.mockRejectedValueOnce(new Error("ServiceNow did not respond.")).mockResolvedValueOnce({ providerRequestId: "snow-1" });- const { consumer, row } = buildConsumer();+ const { consumer, row } = buildConsumer({ delegatedRequest: outboxDelegatedRequest });const { service, rows, elapseBackoff } = buildOutbox(consumer);await service.emit(@@ -471,7 +533,7 @@ describe("delegatedApprovalDispatchConsumer through the outbox", () => {test("closes the request once the outbox has used up its attempts", async () => {dispatch.mockRejectedValue(new Error("ServiceNow did not respond."));- const { consumer, row, approvalUpdate } = buildConsumer();+ const { consumer, row, approvalUpdate } = buildConsumer({ delegatedRequest: outboxDelegatedRequest });const { service, rows, elapseBackoff } = buildOutbox(consumer);await service.emit(@@ -501,4 +563,43 @@ describe("delegatedApprovalDispatchConsumer through the outbox", () => {expect(row()?.status).toBe(DelegatedApprovalRequestStatus.Failed);expect(approvalUpdate).toHaveBeenCalledTimes(1);});++ test("lets a resend replace the event still retrying, so its last attempt neither dispatches nor closes", async () => {+ dispatch.mockRejectedValue(new Error("ServiceNow did not respond."));+ const { consumer, row, update, approvalUpdate } = buildConsumer({ delegatedRequest: outboxDelegatedRequest });+ const { service, rows, elapseBackoff } = buildOutbox(consumer);+ const emitDispatch = (id: string) =>+ service.emit(+ [{ id, type: DELEGATED_APPROVAL_DISPATCH_REQUESTED_EVENT, data: { delegatedRequestId: DELEGATED_REQUEST_ID } }],+ PRIMARY+ );++ await emitDispatch(EVENT_ID);+ for (let attempt = 1; attempt < MAX_OUTBOX_ATTEMPTS; attempt += 1) {+ // eslint-disable-next-line no-await-in-loop -- one drain per attempt+ await service.drain({ consumer: consumer.name });+ elapseBackoff();+ }+ expect(rows[0]).toMatchObject({ status: EventOutboxStatus.Retry, attempts: MAX_OUTBOX_ATTEMPTS - 1 });+ expect(row()?.status).toBe(DelegatedApprovalRequestStatus.FailedDispatch);++ await update(+ { id: DELEGATED_REQUEST_ID, status: DelegatedApprovalRequestStatus.FailedDispatch },+ { dispatchEventId: RESENT_EVENT_ID }+ );+ await emitDispatch(RESENT_EVENT_ID);+ dispatch.mockResolvedValueOnce({ providerRequestId: "snow-1" });++ await service.drain({ consumer: consumer.name });++ expect(dispatch).toHaveBeenCalledTimes(MAX_OUTBOX_ATTEMPTS);+ expect(rows[0]).toMatchObject({ status: EventOutboxStatus.Delivered, attempts: MAX_OUTBOX_ATTEMPTS });+ expect(rows[1]).toMatchObject({ status: EventOutboxStatus.Delivered, attempts: 1 });+ expect(approvalUpdate).not.toHaveBeenCalled();+ expect(row()).toMatchObject({+ status: DelegatedApprovalRequestStatus.WaitingApproval,+ providerRequestId: "snow-1",+ dispatchEventId: RESENT_EVENT_ID+ });+ });});diff --git a/backend/src/services/delegated-approval/dispatch/delegated-approval-dispatch-consumer.ts b/backend/src/services/delegated-approval/dispatch/delegated-approval-dispatch-consumer.tsindex 6a1e5d6098..f4ae9a0b6a 100644--- a/backend/src/services/delegated-approval/dispatch/delegated-approval-dispatch-consumer.ts+++ b/backend/src/services/delegated-approval/dispatch/delegated-approval-dispatch-consumer.ts@@ -66,10 +66,10 @@ export const delegatedApprovalDispatchConsumerFactory = ({appConnectionDAL,kmsService}: TDelegatedApprovalDispatchConsumerDep): IEventConsumer<TDelegatedApprovalDispatchRequestedPayload> => {- const $markDispatchFailed = async (delegatedRequestId: string, statusReason: string) => {+ const $markDispatchFailed = async (delegatedRequestId: string, eventId: string, statusReason: string) => {try {await delegatedApprovalRequestDAL.update(- { id: delegatedRequestId, status: DelegatedApprovalRequestStatus.PendingDispatch },+ { id: delegatedRequestId, status: DelegatedApprovalRequestStatus.PendingDispatch, dispatchEventId: eventId },{status: DelegatedApprovalRequestStatus.FailedDispatch,statusReason: statusReason.slice(0, DELEGATED_APPROVAL_STATUS_REASON_MAX_CHARS)@@ -85,23 +85,27 @@ export const delegatedApprovalDispatchConsumerFactory = ({const $closeAfterExhaustedAttempts = async ({delegatedRequestId,+ eventId,approvalRequestId,statusReason}: {delegatedRequestId: string;+ eventId: string;approvalRequestId: string;statusReason: string;}) => {try {await delegatedApprovalRequestDAL.transaction(async (tx) => {- await delegatedApprovalRequestDAL.update(- { id: delegatedRequestId, status: DelegatedApprovalRequestStatus.PendingDispatch },+ const [closed] = await delegatedApprovalRequestDAL.update(+ { id: delegatedRequestId, status: DelegatedApprovalRequestStatus.PendingDispatch, dispatchEventId: eventId },{status: DelegatedApprovalRequestStatus.Failed,statusReason: statusReason.slice(0, DELEGATED_APPROVAL_STATUS_REASON_MAX_CHARS)},tx);+ if (!closed) return;+await approvalRequestDAL.update({ id: approvalRequestId, status: ApprovalRequestStatus.Pending },{ status: ApprovalRequestStatus.Cancelled },@@ -133,6 +137,7 @@ export const delegatedApprovalDispatchConsumerFactory = ({const $dispatch = async (delegatedRequestId: string,+ eventId: string,isFinalAttempt: boolean): Promise<Omit<TEventConsumerResult, "id">> => {const requests = await $loadRequests(delegatedRequestId);@@ -145,6 +150,13 @@ export const delegatedApprovalDispatchConsumerFactory = ({const { delegatedRequest, approvalRequest, delegatedPolicy } = requests;+ if (delegatedRequest.dispatchEventId !== eventId) {+ logger.info(+ `Delegated approval dispatch was superseded by a newer resend, skipping [delegatedRequestId=${delegatedRequestId}] [eventId=${eventId}]`+ );+ return { status: EventResultStatus.Delivered };+ }+if (delegatedRequest.status !== DelegatedApprovalRequestStatus.PendingDispatch &&delegatedRequest.status !== DelegatedApprovalRequestStatus.FailedDispatch@@ -163,10 +175,16 @@ export const delegatedApprovalDispatchConsumerFactory = ({}if (delegatedRequest.status === DelegatedApprovalRequestStatus.FailedDispatch) {- await delegatedApprovalRequestDAL.update(- { id: delegatedRequestId, status: DelegatedApprovalRequestStatus.FailedDispatch },+ const [reopened] = await delegatedApprovalRequestDAL.update(+ { id: delegatedRequestId, status: DelegatedApprovalRequestStatus.FailedDispatch, dispatchEventId: eventId },{ status: DelegatedApprovalRequestStatus.PendingDispatch, statusReason: null });+ if (!reopened) {+ logger.info(+ `Delegated approval request was resent or closed before this dispatch started, skipping [delegatedRequestId=${delegatedRequestId}] [eventId=${eventId}]`+ );+ return { status: EventResultStatus.Delivered };+ }}try {@@ -261,6 +279,7 @@ export const delegatedApprovalDispatchConsumerFactory = ({if (!isTerminal && isFinalAttempt && approvalRequest) {await $closeAfterExhaustedAttempts({delegatedRequestId,+ eventId,approvalRequestId: approvalRequest.id,statusReason: `${message} Infisical could not send this request after ${MAX_OUTBOX_ATTEMPTS} attempts and closed it. Create a new request.`});@@ -275,6 +294,7 @@ export const delegatedApprovalDispatchConsumerFactory = ({// outbox is still retrying.await $markDispatchFailed(delegatedRequestId,+ eventId,isTerminal? message: `${message} Infisical will retry automatically. If it keeps failing, send the request again.`@@ -302,7 +322,7 @@ export const delegatedApprovalDispatchConsumerFactory = ({};}- return { id, ...(await $dispatch(payload.data.delegatedRequestId, event.isFinalAttempt)) };+ return { id, ...(await $dispatch(payload.data.delegatedRequestId, id, event.isFinalAttempt)) };};const $runEvent = async (event: TEvent): Promise<TEventConsumerResult> => {