Skip to content

Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.

supersed the old event so we don't have two concurrent events trying to retry

adilsondd75b38e1b57
diff --git a/AGENTS.md b/AGENTS.md
index c40b94d8d0..e19e21c08d 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -183,9 +183,11 @@ request goes `cancelled`, and the consumer returns `Failed`. A terminal failure
`failed_dispatch` on every attempt, because an admin can fix the cause and resend. A user resends a
`failed_dispatch` request with `POST /api/v1/access-approvals/requests/:requestId/retry-delegated-dispatch`. The
product's bridge checks who may retry, and `delegatedApprovalService.retryDispatch` checks the request can be
-retried and emits the same event with no transaction. The event id is derived (uuid v5) from the delegated
-request id and that row's `updatedAt`, so repeated emits of one attempt collapse on the outbox's
-`(consumer, eventId)` key, and a later resend still inserts after the status write bumps `updatedAt`. A batch dispatches up to 10 different requests at once;
+retried. A resend supersedes the event still in the outbox rather than running beside it: the row stores the
+live event id in `dispatchEventId`, and a resend writes a new random one (only while the row is
+`failed_dispatch`) and emits the event under it in one transaction. The consumer marks any event whose id no
+longer matches delivered without calling the provider or writing the row, so an older event's last attempt
+can never close a request the user resent, and concurrent resends call the provider once. A batch dispatches up to 10 different requests at once;
events for one request stay serial. A second event for a request that was already delivered is marked delivered
without calling the provider. A delegated request has
no Infisical approvers, so `POST /api/v1/access-approvals/requests/:requestId/cancel-delegated-dispatch` is the only way to close
diff --git a/backend/src/db/migrations/20261028120000_delegated-approval-request-dispatch-event-id.ts b/backend/src/db/migrations/20261028120000_delegated-approval-request-dispatch-event-id.ts
new file mode 100644
index 0000000000..e7cecfafe5
--- /dev/null
+++ b/backend/src/db/migrations/20261028120000_delegated-approval-request-dispatch-event-id.ts
@@ -0,0 +1,24 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+ const hasColumn = await knex.schema.hasColumn(TableName.DelegatedApprovalRequests, "dispatchEventId");
+ if (!hasColumn) {
+ await knex.schema.alterTable(TableName.DelegatedApprovalRequests, (t) => {
+ t.uuid("dispatchEventId").notNullable().defaultTo(knex.fn.uuid());
+ });
+ // The default only backfills existing rows. The id must be the one the caller emits its event under, so
+ // a create that omits it should fail rather than get an id no event carries.
+ await knex.raw('ALTER TABLE ?? ALTER COLUMN "dispatchEventId" DROP DEFAULT', [TableName.DelegatedApprovalRequests]);
+ }
+}
+
+export async function down(knex: Knex): Promise<void> {
+ const hasColumn = await knex.schema.hasColumn(TableName.DelegatedApprovalRequests, "dispatchEventId");
+ if (hasColumn) {
+ await knex.schema.alterTable(TableName.DelegatedApprovalRequests, (t) => {
+ t.dropColumn("dispatchEventId");
+ });
+ }
+}
diff --git a/backend/src/db/schemas/delegated-approval-requests.ts b/backend/src/db/schemas/delegated-approval-requests.ts
index 950f8c0f7e..91d880de98 100644
--- a/backend/src/db/schemas/delegated-approval-requests.ts
+++ b/backend/src/db/schemas/delegated-approval-requests.ts
@@ -17,7 +17,8 @@ export const DelegatedApprovalRequestsSchema = z.object({
statusReason: z.string().nullable().optional(),
providerRequestId: z.string().nullable().optional(),
createdAt: z.date(),
- updatedAt: z.date()
+ updatedAt: z.date(),
+ dispatchEventId: z.string().uuid()
});
export type TDelegatedApprovalRequests = z.infer<typeof DelegatedApprovalRequestsSchema>;
diff --git a/backend/src/services/delegated-approval/delegated-approval-events.ts b/backend/src/services/delegated-approval/delegated-approval-events.ts
index 3c346512d2..5a38abc28a 100644
--- a/backend/src/services/delegated-approval/delegated-approval-events.ts
+++ b/backend/src/services/delegated-approval/delegated-approval-events.ts
@@ -1,14 +1,10 @@
import { Knex } from "knex";
-import { v5 as uuidv5 } from "uuid";
import { z } from "zod";
import { TEventEmitter } from "@app/services/event-outbox/event-outbox-types";
export const DELEGATED_APPROVAL_DISPATCH_REQUESTED_EVENT = "delegated-approval.request.dispatch-requested";
-// Namespaces the event ids derived below, so they can't collide with ids derived the same way elsewhere.
-const DELEGATED_APPROVAL_DISPATCH_REQUESTED_EVENT_ID_NAMESPACE = "95183bf5-9016-4f76-9263-b7682f7e6a14";
-
export const DelegatedApprovalDispatchRequestedPayloadSchema = z.object({
delegatedRequestId: z.string().uuid()
});
@@ -17,25 +13,17 @@ export type TDelegatedApprovalDispatchRequestedPayload = z.infer<
typeof DelegatedApprovalDispatchRequestedPayloadSchema
>;
-// The id is derived from updatedAt, which is the attempt. Repeated retries of one failure share it and
-// the outbox drops all but the first. A request id alone would not do: the previous row is still retained
-// (delivered for a day, failed for 30 days), so a later resend has to insert beside it. The consumer's
-// status write bumps updatedAt, which is what gives that resend a new id.
-export const buildDelegatedApprovalDispatchRequestedEventId = ({
- delegatedRequestId,
- updatedAt
-}: TDelegatedApprovalDispatchRequestedPayload & { updatedAt: Date }): string =>
- uuidv5(`${delegatedRequestId}:${updatedAt.toISOString()}`, DELEGATED_APPROVAL_DISPATCH_REQUESTED_EVENT_ID_NAMESPACE);
-
+// The event id is the request's dispatchEventId. A resend stores a new one, and the consumer skips any event
+// whose id no longer matches, so an older event still in backoff can't dispatch or close the request.
export const emitDelegatedApprovalDispatchRequested = (
eventEmitter: TEventEmitter,
- { delegatedRequestId, updatedAt }: TDelegatedApprovalDispatchRequestedPayload & { updatedAt: Date },
+ { delegatedRequestId, dispatchEventId }: TDelegatedApprovalDispatchRequestedPayload & { dispatchEventId: string },
tx?: Knex
): Promise<void> =>
eventEmitter.emit(
[
{
- id: buildDelegatedApprovalDispatchRequestedEventId({ delegatedRequestId, updatedAt }),
+ id: dispatchEventId,
type: DELEGATED_APPROVAL_DISPATCH_REQUESTED_EVENT,
data: { delegatedRequestId }
}
diff --git a/backend/src/services/delegated-approval/delegated-approval-service.test.ts b/backend/src/services/delegated-approval/delegated-approval-service.test.ts
index b4cf047ef1..2987a7128f 100644
--- a/backend/src/services/delegated-approval/delegated-approval-service.test.ts
+++ b/backend/src/services/delegated-approval/delegated-approval-service.test.ts
@@ -4,18 +4,14 @@ import { BadRequestError } from "@app/lib/errors";
import { ApprovalPolicyType, ApprovalRequestStatus } from "@app/services/approval-policy/approval-policy-enums";
import { DelegatedApprovalPolicyType, DelegatedApprovalRequestStatus } from "./delegated-approval-enums";
-import {
- buildDelegatedApprovalDispatchRequestedEventId,
- DELEGATED_APPROVAL_DISPATCH_REQUESTED_EVENT
-} from "./delegated-approval-events";
+import { DELEGATED_APPROVAL_DISPATCH_REQUESTED_EVENT } from "./delegated-approval-events";
import { delegatedApprovalServiceFactory } from "./delegated-approval-service";
-const UPDATED_AT = new Date("2026-10-23T13:00:00.000Z");
const failedRequest = {
id: "delegated-1",
requestId: "req-1",
status: DelegatedApprovalRequestStatus.FailedDispatch,
- updatedAt: UPDATED_AT
+ dispatchEventId: "44444444-4444-4444-8444-444444444444"
};
const serviceNowPolicy = {
id: "delegated-policy-1",
@@ -36,26 +32,37 @@ const PRIMARY = { primary: true } as never;
const createService = ({
delegatedRequest = failedRequest as unknown,
- delegatedPolicy = serviceNowPolicy as unknown
+ delegatedPolicy = serviceNowPolicy as unknown,
+ resendMatches = true
} = {}) => {
const emit = vi.fn().mockResolvedValue(undefined);
const findOne = vi.fn().mockResolvedValue(delegatedRequest);
const create = vi.fn().mockImplementation(async (row: { requestId: string; status: string }) => ({
id: "delegated-1",
- updatedAt: UPDATED_AT,
...row
}));
+ const update = vi
+ .fn()
+ .mockImplementation(async (_filter: unknown, patch: { dispatchEventId: string }) =>
+ resendMatches ? [{ ...(delegatedRequest as object), ...patch }] : []
+ );
const service = delegatedApprovalServiceFactory({
- delegatedApprovalRequestDAL: { findOne, create, primaryNode: () => PRIMARY },
+ delegatedApprovalRequestDAL: {
+ findOne,
+ create,
+ update,
+ primaryNode: () => PRIMARY,
+ transaction: async (cb: (tx: never) => Promise<unknown>) => cb(TX)
+ },
delegatedApprovalPolicyDAL: { findOne: vi.fn().mockResolvedValue(delegatedPolicy) },
eventEmitter: { emit }
} as never);
- return { service, emit, create, findOne };
+ return { service, emit, create, update, findOne };
};
-const dispatchRequestedEvent = [
+const dispatchRequestedEvent = (id: string) => [
{
- id: buildDelegatedApprovalDispatchRequestedEventId({ delegatedRequestId: "delegated-1", updatedAt: UPDATED_AT }),
+ id,
type: DELEGATED_APPROVAL_DISPATCH_REQUESTED_EVENT,
data: { delegatedRequestId: "delegated-1" }
}
@@ -69,35 +76,55 @@ describe("delegatedApprovalService.createRequest", () => {
expect(request).toMatchObject({ id: "delegated-1", status: DelegatedApprovalRequestStatus.PendingDispatch });
expect(create).toHaveBeenCalledWith(
- { requestId: "req-1", status: DelegatedApprovalRequestStatus.PendingDispatch },
+ {
+ requestId: "req-1",
+ status: DelegatedApprovalRequestStatus.PendingDispatch,
+ dispatchEventId: expect.any(String) as string
+ },
TX
);
expect(emit).toHaveBeenCalledTimes(1);
- expect(emit).toHaveBeenCalledWith(dispatchRequestedEvent, TX);
+ expect(emit).toHaveBeenCalledWith(dispatchRequestedEvent(request.dispatchEventId), TX);
});
});
describe("delegatedApprovalService.retryDispatch", () => {
- test("asks for a new dispatch of a pending request that failed to reach the delegated system", async () => {
- const { service, emit, findOne } = createService();
+ test("supersedes the previous dispatch event with a new one in the same transaction", async () => {
+ const { service, emit, update, findOne } = createService();
- await expect(service.retryDispatch({ approvalRequest: pendingApprovalRequest })).resolves.toEqual({
- delegatedRequest: failedRequest,
- delegatedPolicy: serviceNowPolicy
+ const { delegatedRequest, delegatedPolicy } = await service.retryDispatch({
+ approvalRequest: pendingApprovalRequest
});
+
+ expect(delegatedPolicy).toEqual(serviceNowPolicy);
+ expect(delegatedRequest.dispatchEventId).not.toBe(failedRequest.dispatchEventId);
expect(findOne).toHaveBeenCalledWith({ requestId: "req-1" }, PRIMARY);
- expect(emit).toHaveBeenCalledWith(dispatchRequestedEvent, undefined);
+ expect(update).toHaveBeenCalledWith(
+ { id: "delegated-1", status: DelegatedApprovalRequestStatus.FailedDispatch },
+ { dispatchEventId: delegatedRequest.dispatchEventId },
+ TX
+ );
+ expect(emit).toHaveBeenCalledWith(dispatchRequestedEvent(delegatedRequest.dispatchEventId), TX);
});
- test("emits the same event id when the same failed attempt is sent again", async () => {
+ test("emits a new event id on every resend", async () => {
const { service, emit } = createService();
await service.retryDispatch({ approvalRequest: pendingApprovalRequest });
await service.retryDispatch({ approvalRequest: pendingApprovalRequest });
expect(emit).toHaveBeenCalledTimes(2);
- expect(emit).toHaveBeenNthCalledWith(1, dispatchRequestedEvent, undefined);
- expect(emit).toHaveBeenNthCalledWith(2, dispatchRequestedEvent, undefined);
+ const [[[first]], [[second]]] = emit.mock.calls as [{ id: string }[]][];
+ expect(first.id).not.toBe(second.id);
+ });
+
+ test("refuses a resend when the request left failed_dispatch before it landed", async () => {
+ const { service, emit } = createService({ resendMatches: false });
+
+ await expect(service.retryDispatch({ approvalRequest: pendingApprovalRequest })).rejects.toThrow(
+ "Only requests that failed to reach ServiceNow can be sent again"
+ );
+ expect(emit).not.toHaveBeenCalled();
});
test("refuses a request without a delegated approval", async () => {
@@ -145,11 +172,12 @@ describe("delegatedApprovalService.retryDispatch", () => {
DelegatedApprovalRequestStatus.WaitingApproval,
DelegatedApprovalRequestStatus.Approved
])("refuses a request in the %s state", async (status) => {
- const { service, emit } = createService({ delegatedRequest: { ...failedRequest, status } });
+ const { service, emit, update } = createService({ delegatedRequest: { ...failedRequest, status } });
await expect(service.retryDispatch({ approvalRequest: pendingApprovalRequest })).rejects.toThrow(
"Only requests that failed to reach ServiceNow can be sent again"
);
+ expect(update).not.toHaveBeenCalled();
expect(emit).not.toHaveBeenCalled();
});
});
diff --git a/backend/src/services/delegated-approval/delegated-approval-service.ts b/backend/src/services/delegated-approval/delegated-approval-service.ts
index 063f42988f..5f752d6949 100644
--- a/backend/src/services/delegated-approval/delegated-approval-service.ts
+++ b/backend/src/services/delegated-approval/delegated-approval-service.ts
@@ -1,3 +1,5 @@
+import crypto from "node:crypto";
+
import { Knex } from "knex";
import { DatabaseErrorCode } from "@app/lib/error-codes";
@@ -152,14 +154,15 @@ export const delegatedApprovalServiceFactory = ({
const createRequest = async ({ approvalRequest }: TCreateDelegatedApprovalRequestDTO, tx?: Knex) => {
assertPolicyTypeSupportsDelegatedApproval(approvalRequest.type);
+ const dispatchEventId = crypto.randomUUID();
try {
const request = (await delegatedApprovalRequestDAL.create(
- { requestId: approvalRequest.id, status: DelegatedApprovalRequestStatus.PendingDispatch },
+ { requestId: approvalRequest.id, status: DelegatedApprovalRequestStatus.PendingDispatch, dispatchEventId },
tx
)) as TDelegatedApprovalRequest;
await emitDelegatedApprovalDispatchRequested(
eventEmitter,
- { delegatedRequestId: request.id, updatedAt: request.updatedAt },
+ { delegatedRequestId: request.id, dispatchEventId },
tx
);
return request;
@@ -210,18 +213,34 @@ export const delegatedApprovalServiceFactory = ({
});
}
+ const notResendableError = new BadRequestError({
+ message: `Only requests that failed to reach ${DELEGATED_APPROVAL_NAME_MAP[delegatedPolicy.type]} can be sent again`
+ });
+
if (delegatedRequest.status !== DelegatedApprovalRequestStatus.FailedDispatch) {
- throw new BadRequestError({
- message: `Only requests that failed to reach ${DELEGATED_APPROVAL_NAME_MAP[delegatedPolicy.type]} can be sent again`
- });
+ throw notResendableError;
}
- await emitDelegatedApprovalDispatchRequested(eventEmitter, {
- delegatedRequestId: delegatedRequest.id,
- updatedAt: delegatedRequest.updatedAt
+ // Conditional on failed_dispatch so it serializes with the consumer reopening the row for the event this
+ // replaces: whichever takes the row first wins, and the other backs off.
+ const dispatchEventId = crypto.randomUUID();
+ const resentRequest = await delegatedApprovalRequestDAL.transaction(async (tx) => {
+ const [resent] = await delegatedApprovalRequestDAL.update(
+ { id: delegatedRequest.id, status: DelegatedApprovalRequestStatus.FailedDispatch },
+ { dispatchEventId },
+ tx
+ );
+ if (!resent) throw notResendableError;
+
+ await emitDelegatedApprovalDispatchRequested(
+ eventEmitter,
+ { delegatedRequestId: resent.id, dispatchEventId },
+ tx
+ );
+ return resent as TDelegatedApprovalRequest;
});
- return { delegatedRequest, delegatedPolicy };
+ return { delegatedRequest: resentRequest, delegatedPolicy };
};
const updateRequest = async ({ requestId, status, statusReason }: TUpdateDelegatedApprovalRequestDTO, tx?: Knex) => {
diff --git a/backend/src/services/delegated-approval/dispatch/delegated-approval-dispatch-consumer.test.ts b/backend/src/services/delegated-approval/dispatch/delegated-approval-dispatch-consumer.test.ts
index bd92e61c31..9304339089 100644
--- a/backend/src/services/delegated-approval/dispatch/delegated-approval-dispatch-consumer.test.ts
+++ b/backend/src/services/delegated-approval/dispatch/delegated-approval-dispatch-consumer.test.ts
@@ -51,6 +51,7 @@ vi.mock("@app/services/app-connection/app-connection-fns", () => ({
const DELEGATED_REQUEST_ID = "11111111-1111-1111-1111-111111111111";
const EVENT_ID = "44444444-4444-4444-8444-444444444444";
+const RESENT_EVENT_ID = "55555555-5555-4555-8555-555555555555";
const ORG_ID = "22222222-2222-2222-2222-222222222222";
const RETRY_REASON =
"ServiceNow did not respond. Infisical will retry automatically. If it keeps failing, send the request again.";
@@ -61,7 +62,8 @@ const pendingDelegatedRequest = {
status: DelegatedApprovalRequestStatus.PendingDispatch,
statusReason: null as string | null,
providerRequestId: null as string | null,
- encryptedCallbackToken: null
+ encryptedCallbackToken: null,
+ dispatchEventId: "event-1"
};
type TRow = typeof pendingDelegatedRequest;
@@ -284,7 +286,8 @@ describe("delegatedApprovalDispatchConsumer", () => {
findById: async (id: string) => ({
...pendingDelegatedRequest,
id,
- requestId: id === DELEGATED_REQUEST_ID ? "req-1" : "req-2"
+ requestId: id === DELEGATED_REQUEST_ID ? "req-1" : "req-2",
+ dispatchEventId: id === DELEGATED_REQUEST_ID ? "event-1" : "event-2"
})
});
@@ -305,45 +308,102 @@ describe("delegatedApprovalDispatchConsumer", () => {
]);
});
- test("dispatches a second event for the same request only after the first settles", async () => {
+ test("handles a second event for the same request only after the first settles", async () => {
let releaseFirst: () => void = () => {};
const firstHeld = new Promise<void>((resolve) => {
releaseFirst = resolve;
});
- let firstSettledAt = 0;
- let secondStartedAt = 0;
dispatch.mockImplementation(async () => {
- if (dispatch.mock.calls.length === 1) {
- await firstHeld;
- firstSettledAt = Date.now();
- throw new Error("ServiceNow did not respond.");
- }
- secondStartedAt = Date.now();
- return { providerRequestId: "snow-2" };
+ await firstHeld;
+ throw new Error("ServiceNow did not respond.");
});
- const { consumer } = buildConsumer();
- const pending = consumer.handle([makeEvent({ id: "event-1" }), makeEvent({ id: "event-2" })]);
+ let readRow: () => TRow | null = () => null;
+ const findById = vi.fn(async () => readRow());
+ const built = buildConsumer({ findById });
+ readRow = built.row;
+
+ const pending = built.consumer.handle([makeEvent({ id: "event-1" }), makeEvent({ id: "event-2" })]);
try {
await vi.waitFor(() => expect(dispatch).toHaveBeenCalledTimes(1));
await new Promise((resolve) => {
setTimeout(resolve, 30);
});
- expect(dispatch).toHaveBeenCalledTimes(1);
+ expect(findById).toHaveBeenCalledTimes(1);
} finally {
releaseFirst();
}
const results = await pending;
- expect(dispatch).toHaveBeenCalledTimes(2);
- expect(firstSettledAt).toBeGreaterThan(0);
- expect(secondStartedAt).toBeGreaterThanOrEqual(firstSettledAt);
+ expect(findById).toHaveBeenCalledTimes(2);
expect(results.map((result) => result.status)).toEqual([EventResultStatus.Retry, EventResultStatus.Delivered]);
});
+ test("skips a superseded event batched with the event that replaced it", async () => {
+ dispatch.mockResolvedValueOnce({ providerRequestId: "snow-2" });
+ const { consumer, row } = buildConsumer({
+ delegatedRequest: { ...pendingDelegatedRequest, dispatchEventId: "event-2" }
+ });
+
+ const results = await consumer.handle([makeEvent({ id: "event-1" }), makeEvent({ id: "event-2" })]);
+
+ expect(results).toEqual([
+ { id: "event-1", status: EventResultStatus.Delivered },
+ { id: "event-2", status: EventResultStatus.Delivered }
+ ]);
+ expect(dispatch).toHaveBeenCalledTimes(1);
+ expect(row()).toMatchObject({
+ status: DelegatedApprovalRequestStatus.WaitingApproval,
+ providerRequestId: "snow-2"
+ });
+ });
+
+ test.each([
+ ["an earlier attempt", false],
+ ["its last attempt", true]
+ ])(
+ "marks an event superseded by a resend delivered on %s without dispatching or closing the request",
+ async (_, isFinalAttempt) => {
+ const resent = {
+ ...pendingDelegatedRequest,
+ status: DelegatedApprovalRequestStatus.FailedDispatch,
+ statusReason: RETRY_REASON,
+ dispatchEventId: "event-2"
+ };
+ const { consumer, row, update, approvalUpdate } = buildConsumer({ delegatedRequest: resent });
+
+ const results = await consumer.handle([makeEvent({ id: "event-1", isFinalAttempt })]);
+
+ expect(results).toEqual([{ id: "event-1", status: EventResultStatus.Delivered }]);
+ expect(dispatch).not.toHaveBeenCalled();
+ expect(update).not.toHaveBeenCalled();
+ expect(approvalUpdate).not.toHaveBeenCalled();
+ expect(row()).toEqual(resent);
+ }
+ );
+
+ test("stops when a resend lands between loading the request and reopening it", async () => {
+ const resent = {
+ ...pendingDelegatedRequest,
+ status: DelegatedApprovalRequestStatus.FailedDispatch,
+ statusReason: RETRY_REASON,
+ dispatchEventId: "event-2"
+ };
+ const { consumer, row } = buildConsumer({
+ delegatedRequest: resent,
+ findById: async () => ({ ...resent, dispatchEventId: "event-1" })
+ });
+
+ const results = await consumer.handle([makeEvent({ id: "event-1" })]);
+
+ expect(results).toEqual([{ id: "event-1", status: EventResultStatus.Delivered }]);
+ expect(dispatch).not.toHaveBeenCalled();
+ expect(row()).toEqual(resent);
+ });
+
test("fails an unreadable payload terminally", async () => {
const { consumer } = buildConsumer();
@@ -358,6 +418,8 @@ describe("delegatedApprovalDispatchConsumer", () => {
// Why this consumer exists: a dispatch that fails is kept by the outbox and tried again, instead of
// depending on an enqueue that ran after the request committed and could be lost with it.
describe("delegatedApprovalDispatchConsumer through the outbox", () => {
+ const outboxDelegatedRequest = { ...pendingDelegatedRequest, dispatchEventId: EVENT_ID };
+
const buildOutbox = (consumer: ReturnType<typeof buildConsumer>["consumer"]) => {
const registry = eventOutboxRegistryFactory();
registry.register(consumer);
@@ -429,7 +491,7 @@ describe("delegatedApprovalDispatchConsumer through the outbox", () => {
dispatch
.mockRejectedValueOnce(new Error("ServiceNow did not respond."))
.mockResolvedValueOnce({ providerRequestId: "snow-1" });
- const { consumer, row } = buildConsumer();
+ const { consumer, row } = buildConsumer({ delegatedRequest: outboxDelegatedRequest });
const { service, rows, elapseBackoff } = buildOutbox(consumer);
await service.emit(
@@ -471,7 +533,7 @@ describe("delegatedApprovalDispatchConsumer through the outbox", () => {
test("closes the request once the outbox has used up its attempts", async () => {
dispatch.mockRejectedValue(new Error("ServiceNow did not respond."));
- const { consumer, row, approvalUpdate } = buildConsumer();
+ const { consumer, row, approvalUpdate } = buildConsumer({ delegatedRequest: outboxDelegatedRequest });
const { service, rows, elapseBackoff } = buildOutbox(consumer);
await service.emit(
@@ -501,4 +563,43 @@ describe("delegatedApprovalDispatchConsumer through the outbox", () => {
expect(row()?.status).toBe(DelegatedApprovalRequestStatus.Failed);
expect(approvalUpdate).toHaveBeenCalledTimes(1);
});
+
+ test("lets a resend replace the event still retrying, so its last attempt neither dispatches nor closes", async () => {
+ dispatch.mockRejectedValue(new Error("ServiceNow did not respond."));
+ const { consumer, row, update, approvalUpdate } = buildConsumer({ delegatedRequest: outboxDelegatedRequest });
+ const { service, rows, elapseBackoff } = buildOutbox(consumer);
+ const emitDispatch = (id: string) =>
+ service.emit(
+ [{ id, type: DELEGATED_APPROVAL_DISPATCH_REQUESTED_EVENT, data: { delegatedRequestId: DELEGATED_REQUEST_ID } }],
+ PRIMARY
+ );
+
+ await emitDispatch(EVENT_ID);
+ for (let attempt = 1; attempt < MAX_OUTBOX_ATTEMPTS; attempt += 1) {
+ // eslint-disable-next-line no-await-in-loop -- one drain per attempt
+ await service.drain({ consumer: consumer.name });
+ elapseBackoff();
+ }
+ expect(rows[0]).toMatchObject({ status: EventOutboxStatus.Retry, attempts: MAX_OUTBOX_ATTEMPTS - 1 });
+ expect(row()?.status).toBe(DelegatedApprovalRequestStatus.FailedDispatch);
+
+ await update(
+ { id: DELEGATED_REQUEST_ID, status: DelegatedApprovalRequestStatus.FailedDispatch },
+ { dispatchEventId: RESENT_EVENT_ID }
+ );
+ await emitDispatch(RESENT_EVENT_ID);
+ dispatch.mockResolvedValueOnce({ providerRequestId: "snow-1" });
+
+ await service.drain({ consumer: consumer.name });
+
+ expect(dispatch).toHaveBeenCalledTimes(MAX_OUTBOX_ATTEMPTS);
+ expect(rows[0]).toMatchObject({ status: EventOutboxStatus.Delivered, attempts: MAX_OUTBOX_ATTEMPTS });
+ expect(rows[1]).toMatchObject({ status: EventOutboxStatus.Delivered, attempts: 1 });
+ expect(approvalUpdate).not.toHaveBeenCalled();
+ expect(row()).toMatchObject({
+ status: DelegatedApprovalRequestStatus.WaitingApproval,
+ providerRequestId: "snow-1",
+ dispatchEventId: RESENT_EVENT_ID
+ });
+ });
});
diff --git a/backend/src/services/delegated-approval/dispatch/delegated-approval-dispatch-consumer.ts b/backend/src/services/delegated-approval/dispatch/delegated-approval-dispatch-consumer.ts
index 6a1e5d6098..f4ae9a0b6a 100644
--- a/backend/src/services/delegated-approval/dispatch/delegated-approval-dispatch-consumer.ts
+++ b/backend/src/services/delegated-approval/dispatch/delegated-approval-dispatch-consumer.ts
@@ -66,10 +66,10 @@ export const delegatedApprovalDispatchConsumerFactory = ({
appConnectionDAL,
kmsService
}: TDelegatedApprovalDispatchConsumerDep): IEventConsumer<TDelegatedApprovalDispatchRequestedPayload> => {
- const $markDispatchFailed = async (delegatedRequestId: string, statusReason: string) => {
+ const $markDispatchFailed = async (delegatedRequestId: string, eventId: string, statusReason: string) => {
try {
await delegatedApprovalRequestDAL.update(
- { id: delegatedRequestId, status: DelegatedApprovalRequestStatus.PendingDispatch },
+ { id: delegatedRequestId, status: DelegatedApprovalRequestStatus.PendingDispatch, dispatchEventId: eventId },
{
status: DelegatedApprovalRequestStatus.FailedDispatch,
statusReason: statusReason.slice(0, DELEGATED_APPROVAL_STATUS_REASON_MAX_CHARS)
@@ -85,23 +85,27 @@ export const delegatedApprovalDispatchConsumerFactory = ({
const $closeAfterExhaustedAttempts = async ({
delegatedRequestId,
+ eventId,
approvalRequestId,
statusReason
}: {
delegatedRequestId: string;
+ eventId: string;
approvalRequestId: string;
statusReason: string;
}) => {
try {
await delegatedApprovalRequestDAL.transaction(async (tx) => {
- await delegatedApprovalRequestDAL.update(
- { id: delegatedRequestId, status: DelegatedApprovalRequestStatus.PendingDispatch },
+ const [closed] = await delegatedApprovalRequestDAL.update(
+ { id: delegatedRequestId, status: DelegatedApprovalRequestStatus.PendingDispatch, dispatchEventId: eventId },
{
status: DelegatedApprovalRequestStatus.Failed,
statusReason: statusReason.slice(0, DELEGATED_APPROVAL_STATUS_REASON_MAX_CHARS)
},
tx
);
+ if (!closed) return;
+
await approvalRequestDAL.update(
{ id: approvalRequestId, status: ApprovalRequestStatus.Pending },
{ status: ApprovalRequestStatus.Cancelled },
@@ -133,6 +137,7 @@ export const delegatedApprovalDispatchConsumerFactory = ({
const $dispatch = async (
delegatedRequestId: string,
+ eventId: string,
isFinalAttempt: boolean
): Promise<Omit<TEventConsumerResult, "id">> => {
const requests = await $loadRequests(delegatedRequestId);
@@ -145,6 +150,13 @@ export const delegatedApprovalDispatchConsumerFactory = ({
const { delegatedRequest, approvalRequest, delegatedPolicy } = requests;
+ if (delegatedRequest.dispatchEventId !== eventId) {
+ logger.info(
+ `Delegated approval dispatch was superseded by a newer resend, skipping [delegatedRequestId=${delegatedRequestId}] [eventId=${eventId}]`
+ );
+ return { status: EventResultStatus.Delivered };
+ }
+
if (
delegatedRequest.status !== DelegatedApprovalRequestStatus.PendingDispatch &&
delegatedRequest.status !== DelegatedApprovalRequestStatus.FailedDispatch
@@ -163,10 +175,16 @@ export const delegatedApprovalDispatchConsumerFactory = ({
}
if (delegatedRequest.status === DelegatedApprovalRequestStatus.FailedDispatch) {
- await delegatedApprovalRequestDAL.update(
- { id: delegatedRequestId, status: DelegatedApprovalRequestStatus.FailedDispatch },
+ const [reopened] = await delegatedApprovalRequestDAL.update(
+ { id: delegatedRequestId, status: DelegatedApprovalRequestStatus.FailedDispatch, dispatchEventId: eventId },
{ status: DelegatedApprovalRequestStatus.PendingDispatch, statusReason: null }
);
+ if (!reopened) {
+ logger.info(
+ `Delegated approval request was resent or closed before this dispatch started, skipping [delegatedRequestId=${delegatedRequestId}] [eventId=${eventId}]`
+ );
+ return { status: EventResultStatus.Delivered };
+ }
}
try {
@@ -261,6 +279,7 @@ export const delegatedApprovalDispatchConsumerFactory = ({
if (!isTerminal && isFinalAttempt && approvalRequest) {
await $closeAfterExhaustedAttempts({
delegatedRequestId,
+ eventId,
approvalRequestId: approvalRequest.id,
statusReason: `${message} Infisical could not send this request after ${MAX_OUTBOX_ATTEMPTS} attempts and closed it. Create a new request.`
});
@@ -275,6 +294,7 @@ export const delegatedApprovalDispatchConsumerFactory = ({
// outbox is still retrying.
await $markDispatchFailed(
delegatedRequestId,
+ eventId,
isTerminal
? message
: `${message} Infisical will retry automatically. If it keeps failing, send the request again.`
@@ -302,7 +322,7 @@ export const delegatedApprovalDispatchConsumerFactory = ({
};
}
- return { id, ...(await $dispatch(payload.data.delegatedRequestId, event.isFinalAttempt)) };
+ return { id, ...(await $dispatch(payload.data.delegatedRequestId, id, event.isFinalAttempt)) };
};
const $runEvent = async (event: TEvent): Promise<TEventConsumerResult> => {