Skip to content

Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.

fix(secret-rotation): retry Azure client secret revoke and clean up new secret on failure

Claude472fde12a71a
diff --git a/backend/src/ee/services/secret-rotation-v2/azure-client-secret/azure-client-secret-rotation-fns.test.ts b/backend/src/ee/services/secret-rotation-v2/azure-client-secret/azure-client-secret-rotation-fns.test.ts
new file mode 100644
index 0000000000..5591bd5380
--- /dev/null
+++ b/backend/src/ee/services/secret-rotation-v2/azure-client-secret/azure-client-secret-rotation-fns.test.ts
@@ -0,0 +1,191 @@
+/* eslint-disable @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-argument */
+import { AxiosError } from "axios";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+// `vi.mock` factories are hoisted above imports, so the spies they reference must come from `vi.hoisted`.
+const { getMock, postMock } = vi.hoisted(() => ({
+ getMock: vi.fn(),
+ postMock: vi.fn()
+}));
+
+vi.mock("@app/lib/config/request", () => ({
+ request: {
+ get: (...args: unknown[]) => (getMock as any)(...args),
+ post: (...args: unknown[]) => (postMock as any)(...args)
+ }
+}));
+vi.mock("@app/lib/logger", () => ({
+ logger: { error: vi.fn(), info: vi.fn(), warn: vi.fn(), debug: vi.fn() }
+}));
+vi.mock("@app/services/app-connection/azure-client-secrets/azure-client-secrets-connection-fns", () => ({
+ getAzureConnectionAccessToken: vi.fn(async () => "access-token")
+}));
+
+// eslint-disable-next-line import/first
+import { azureClientSecretRotationFactory } from "./azure-client-secret-rotation-fns";
+
+const OBJECT_ID = "app-object-id";
+const OLD_KEY_ID = "old-key-id";
+const NEW_KEY_ID = "new-key-id";
+
+const concurrentRequestError = () =>
+ new AxiosError("Request failed with status code 400", "ERR_BAD_REQUEST", undefined, undefined, {
+ status: 400,
+ statusText: "Bad Request",
+ headers: {},
+ config: {} as any,
+ data: {
+ error: {
+ code: "Request_BadRequest",
+ message: "Error due to concurrent requests being made to the tenant. Please wait briefly and retry."
+ }
+ }
+ });
+
+const forbiddenError = () =>
+ new AxiosError("Request failed with status code 403", "ERR_BAD_REQUEST", undefined, undefined, {
+ status: 403,
+ statusText: "Forbidden",
+ headers: {},
+ config: {} as any,
+ data: {
+ error: { code: "Authorization_RequestDenied", message: "Insufficient privileges to complete the operation." }
+ }
+ });
+
+const isRemovePassword = (url: string) => url.endsWith("/removePassword");
+
+const removePasswordCalls = (keyId?: string) =>
+ postMock.mock.calls.filter(([url, body]) => isRemovePassword(url) && (keyId === undefined || body.keyId === keyId));
+
+const addPasswordCalls = () => postMock.mock.calls.filter(([url]) => url.endsWith("/addPassword"));
+
+// removePassword behaviour is keyed by keyId so a test can fail the old secret's revoke while letting cleanup succeed.
+const mockGraph = (removePassword: Record<string, () => Promise<unknown>>) => {
+ getMock.mockResolvedValue({ data: { value: [{ keyId: OLD_KEY_ID }, { keyId: NEW_KEY_ID }] } });
+ postMock.mockImplementation(async (url: string, body: any) => {
+ if (url.endsWith("/addPassword")) return { data: { secretText: "new-secret", keyId: NEW_KEY_ID } };
+ if (isRemovePassword(url)) return removePassword[body.keyId]();
+ throw new Error(`unexpected request to ${url}`);
+ });
+};
+
+const failTimesThenSucceed = (failures: number) => {
+ let remaining = failures;
+ return async () => {
+ if (remaining > 0) {
+ remaining -= 1;
+ throw concurrentRequestError();
+ }
+ return { data: {} };
+ };
+};
+
+const alwaysFail = (error: () => Error) => async () => {
+ throw error();
+};
+
+const succeed = async () => ({ data: {} });
+
+const makeFactory = () =>
+ azureClientSecretRotationFactory(
+ {
+ connection: { id: "connection-id", credentials: { tenantId: "tenant-id" } },
+ parameters: { objectId: OBJECT_ID, clientId: "client-id" },
+ secretsMapping: { clientSecret: "AZURE_CLIENT_SECRET", clientId: "AZURE_CLIENT_ID" },
+ rotationInterval: 30
+ } as any,
+ {} as any,
+ {} as any,
+ {} as any,
+ {} as any,
+ {} as any
+ );
+
+// The backoff sleeps are real timers, so the rotation is driven to completion under fake timers.
+const rotate = async (callback: (credentials: unknown) => Promise<unknown>) => {
+ const rotation = makeFactory().rotateCredentials(
+ { keyId: OLD_KEY_ID, clientId: "client-id", clientSecret: "old-secret" } as any,
+ callback as any,
+ {} as any
+ );
+ // Attach the handler before advancing timers so a rejection is never left unobserved.
+ const settled = rotation.then(
+ (value) => ({ status: "fulfilled" as const, value }),
+ (reason: unknown) => ({ status: "rejected" as const, reason })
+ );
+ await vi.runAllTimersAsync();
+ return settled;
+};
+
+describe("azureClientSecretRotationFactory.rotateCredentials", () => {
+ beforeEach(() => {
+ vi.useFakeTimers();
+ getMock.mockReset();
+ postMock.mockReset();
+ });
+
+ afterEach(() => {
+ vi.useRealTimers();
+ });
+
+ it("retries the revoke on Graph's concurrent request error and persists the new secret once it succeeds", async () => {
+ mockGraph({ [OLD_KEY_ID]: failTimesThenSucceed(2) });
+ const callback = vi.fn(async (credentials: unknown) => credentials);
+
+ const result = await rotate(callback);
+
+ expect(result.status).toBe("fulfilled");
+ expect(addPasswordCalls()).toHaveLength(1);
+ expect(removePasswordCalls(OLD_KEY_ID)).toHaveLength(3);
+ expect(removePasswordCalls(NEW_KEY_ID)).toHaveLength(0);
+ expect(callback).toHaveBeenCalledTimes(1);
+ expect(callback).toHaveBeenCalledWith({ clientSecret: "new-secret", keyId: NEW_KEY_ID, clientId: "client-id" });
+ });
+
+ it("removes the newly created secret and fails the rotation when the revoke keeps failing", async () => {
+ mockGraph({ [OLD_KEY_ID]: alwaysFail(concurrentRequestError), [NEW_KEY_ID]: succeed });
+ const callback = vi.fn(async (credentials: unknown) => credentials);
+
+ const result = await rotate(callback);
+
+ expect(result.status).toBe("rejected");
+ expect((result as { reason: Error }).reason.message).toContain(
+ `Failed to remove client secret with keyId ${OLD_KEY_ID} from app ${OBJECT_ID}`
+ );
+ expect((result as { reason: Error }).reason.message).not.toContain("removed manually");
+ expect(addPasswordCalls()).toHaveLength(1);
+ expect(removePasswordCalls(OLD_KEY_ID)).toHaveLength(4);
+ expect(removePasswordCalls(NEW_KEY_ID)).toHaveLength(1);
+ expect(callback).not.toHaveBeenCalled();
+ });
+
+ it("reports the orphaned new secret when the cleanup fails too", async () => {
+ mockGraph({ [OLD_KEY_ID]: alwaysFail(concurrentRequestError), [NEW_KEY_ID]: alwaysFail(concurrentRequestError) });
+ const callback = vi.fn(async (credentials: unknown) => credentials);
+
+ const result = await rotate(callback);
+
+ expect(result.status).toBe("rejected");
+ const { message } = (result as { reason: Error }).reason;
+ expect(message).toContain(`Failed to remove client secret with keyId ${OLD_KEY_ID} from app ${OBJECT_ID}`);
+ expect(message).toContain(`keyId ${NEW_KEY_ID} could not be cleaned up from app ${OBJECT_ID}`);
+ expect(message).toContain("removed manually");
+ expect(removePasswordCalls(OLD_KEY_ID)).toHaveLength(4);
+ expect(removePasswordCalls(NEW_KEY_ID)).toHaveLength(4);
+ expect(callback).not.toHaveBeenCalled();
+ });
+
+ it("does not retry revoke failures other than the concurrent request error", async () => {
+ mockGraph({ [OLD_KEY_ID]: alwaysFail(forbiddenError), [NEW_KEY_ID]: succeed });
+ const callback = vi.fn(async (credentials: unknown) => credentials);
+
+ const result = await rotate(callback);
+
+ expect(result.status).toBe("rejected");
+ expect((result as { reason: Error }).reason.message).toContain("Insufficient privileges");
+ expect(removePasswordCalls(OLD_KEY_ID)).toHaveLength(1);
+ expect(removePasswordCalls(NEW_KEY_ID)).toHaveLength(1);
+ expect(callback).not.toHaveBeenCalled();
+ });
+});
diff --git a/backend/src/ee/services/secret-rotation-v2/azure-client-secret/azure-client-secret-rotation-fns.ts b/backend/src/ee/services/secret-rotation-v2/azure-client-secret/azure-client-secret-rotation-fns.ts
index 0e9c4567b4..d55a2de65f 100644
--- a/backend/src/ee/services/secret-rotation-v2/azure-client-secret/azure-client-secret-rotation-fns.ts
+++ b/backend/src/ee/services/secret-rotation-v2/azure-client-secret/azure-client-secret-rotation-fns.ts
@@ -16,6 +16,7 @@ import {
} from "@app/ee/services/secret-rotation-v2/secret-rotation-v2-types";
import { request } from "@app/lib/config/request";
import { BadRequestError } from "@app/lib/errors";
+import { logger } from "@app/lib/logger";
import { getAzureConnectionAccessToken } from "@app/services/app-connection/azure-client-secrets/azure-client-secrets-connection-fns";
const GRAPH_API_BASE = "https://graph.microsoft.com/v1.0";
@@ -23,12 +24,40 @@ const GRAPH_API_BASE = "https://graph.microsoft.com/v1.0";
type AzureErrorResponse = { error: { message: string } };
const EXPIRY_PADDING_IN_DAYS = 3;
+const AZURE_CONCURRENT_REQUEST_MAX_RETRIES = 3;
+const AZURE_CONCURRENT_REQUEST_BASE_DELAY_MS = 2000;
-const sleep = async () =>
- new Promise((resolve) => {
- setTimeout(resolve, 1000);
+const sleep = async (ms = 1000) =>
+ new Promise<void>((resolve) => {
+ setTimeout(resolve, ms);
});
+const getAzureErrorMessage = (error: AxiosError): string | undefined => {
+ const message = (error.response?.data as Partial<AzureErrorResponse> | undefined)?.error?.message;
+ return typeof message === "string" ? message : undefined;
+};
+
+const getErrorMessage = (error: unknown): string => (error instanceof Error ? error.message : "Unknown error");
+
+// Graph rejects back-to-back writes to the same app registration with a transient "concurrent requests" error
+const isAzureConcurrentRequestError = (error: unknown): boolean =>
+ error instanceof AxiosError && (getAzureErrorMessage(error) ?? "").toLowerCase().includes("concurrent requests");
+
+const withAzureConcurrentRequestRetry = async <T>(fn: () => Promise<T>, context: string, attempt = 0): Promise<T> => {
+ try {
+ return await fn();
+ } catch (error: unknown) {
+ if (!isAzureConcurrentRequestError(error) || attempt >= AZURE_CONCURRENT_REQUEST_MAX_RETRIES) throw error;
+
+ const delay = AZURE_CONCURRENT_REQUEST_BASE_DELAY_MS * 2 ** attempt;
+ logger.info(
+ `secretRotation: Azure concurrent request error on ${context}, retrying in ${delay}ms (attempt ${attempt + 1}/${AZURE_CONCURRENT_REQUEST_MAX_RETRIES})`
+ );
+ await sleep(delay);
+ return withAzureConcurrentRequestRetry(fn, context, attempt + 1);
+ }
+};
+
export const azureClientSecretRotationFactory: TRotationFactory<
TAzureClientSecretRotationWithConnection,
TAzureClientSecretRotationGeneratedCredentials
@@ -84,15 +113,7 @@ export const azureClientSecretRotationFactory: TRotationFactory<
};
} catch (error: unknown) {
if (error instanceof AxiosError) {
- let message;
- if (
- error.response?.data &&
- typeof error.response.data === "object" &&
- "error" in error.response.data &&
- typeof (error.response.data as AzureErrorResponse).error.message === "string"
- ) {
- message = (error.response.data as AzureErrorResponse).error.message;
- }
+ const message = getAzureErrorMessage(error);
throw new BadRequestError({
message: `Failed to add client secret to Azure app ${objectId}: ${
message || error.message || "Unknown error"
@@ -113,25 +134,21 @@ export const azureClientSecretRotationFactory: TRotationFactory<
const endpoint = `${GRAPH_API_BASE}/applications/${objectId}/passwordCredentials`;
try {
- const { data } = await request.get<{ value: Array<{ keyId: string }> }>(endpoint, {
- headers: {
- Authorization: `Bearer ${accessToken}`,
- "Content-Type": "application/json"
- }
- });
+ const { data } = await withAzureConcurrentRequestRetry(
+ () =>
+ request.get<{ value: Array<{ keyId: string }> }>(endpoint, {
+ headers: {
+ Authorization: `Bearer ${accessToken}`,
+ "Content-Type": "application/json"
+ }
+ }),
+ "list credentials"
+ );
return data.value?.some((credential) => credential.keyId === keyId) || false;
} catch (error: unknown) {
if (error instanceof AxiosError) {
- let message;
- if (
- error.response?.data &&
- typeof error.response.data === "object" &&
- "error" in error.response.data &&
- typeof (error.response.data as AzureErrorResponse).error.message === "string"
- ) {
- message = (error.response.data as AzureErrorResponse).error.message;
- }
+ const message = getAzureErrorMessage(error);
throw new BadRequestError({
message: `Failed to check credential existence for app ${objectId}: ${
message || error.message || "Unknown error"
@@ -144,42 +161,28 @@ export const azureClientSecretRotationFactory: TRotationFactory<
}
};
- /**
- * Revokes a client secret from the Azure app using its keyId.
- * First checks if the credential exists before attempting revocation.
- */
- const revokeCredential = async (keyId: string) => {
- // Check if credential exists before attempting revocation
- const exists = await credentialExists(keyId);
- if (!exists) {
- return; // Credential doesn't exist, nothing to revoke
- }
-
+ const $removeClientSecret = async (keyId: string) => {
const accessToken = await getAzureConnectionAccessToken(connection.id, appConnectionDAL, kmsService);
const endpoint = `${GRAPH_API_BASE}/applications/${objectId}/removePassword`;
try {
- await request.post(
- endpoint,
- { keyId },
- {
- headers: {
- Authorization: `Bearer ${accessToken}`,
- "Content-Type": "application/json"
- }
- }
+ await withAzureConcurrentRequestRetry(
+ () =>
+ request.post(
+ endpoint,
+ { keyId },
+ {
+ headers: {
+ Authorization: `Bearer ${accessToken}`,
+ "Content-Type": "application/json"
+ }
+ }
+ ),
+ "remove credential"
);
} catch (error: unknown) {
if (error instanceof AxiosError) {
- let message;
- if (
- error.response?.data &&
- typeof error.response.data === "object" &&
- "error" in error.response.data &&
- typeof (error.response.data as AzureErrorResponse).error.message === "string"
- ) {
- message = (error.response.data as AzureErrorResponse).error.message;
- }
+ const message = getAzureErrorMessage(error);
throw new BadRequestError({
message: `Failed to remove client secret with keyId ${keyId} from app ${objectId}: ${
message || error.message || "Unknown error"
@@ -192,6 +195,19 @@ export const azureClientSecretRotationFactory: TRotationFactory<
}
};
+ /**
+ * Revokes a client secret from the Azure app using its keyId.
+ * First checks if the credential exists before attempting revocation.
+ */
+ const revokeCredential = async (keyId: string) => {
+ const exists = await credentialExists(keyId);
+ if (!exists) {
+ return; // Credential doesn't exist, nothing to revoke
+ }
+
+ await $removeClientSecret(keyId);
+ };
+
/**
* Issues a new set of credentials.
*/
@@ -233,7 +249,21 @@ export const azureClientSecretRotationFactory: TRotationFactory<
) => {
const newCredentials = await $rotateClientSecret();
if (oldCredentials?.keyId) {
- await revokeCredential(oldCredentials.keyId);
+ try {
+ await revokeCredential(oldCredentials.keyId);
+ } catch (revokeError: unknown) {
+ // only two credentials are tracked, so persisting the new secret now would drop the old keyId from tracking
+ try {
+ await sleep(AZURE_CONCURRENT_REQUEST_BASE_DELAY_MS);
+ await $removeClientSecret(newCredentials.keyId);
+ } catch (cleanupError: unknown) {
+ const cleanupMessage = `The newly created client secret with keyId ${newCredentials.keyId} could not be cleaned up from app ${objectId} and may need to be removed manually`;
+ throw new BadRequestError({
+ message: `${getErrorMessage(revokeError)} ${cleanupMessage}: ${getErrorMessage(cleanupError)}`
+ });
+ }
+ throw revokeError;
+ }
}
return callback(newCredentials);