Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.
TypeScript acmeagentsai-agentscertificate-managementdevsecopsenvironment-variableskubernetesopen-sourcepampkipostgresprivate-caprivileged-access-managementsecret-managementsecret-managersecretssecrets-managementsecuritysecurity-toolsvault
fix(secret-rotation): retry Azure client secret revoke and clean up new secret on failure
Claude472fde12a71a
diff --git a/backend/src/ee/services/secret-rotation-v2/azure-client-secret/azure-client-secret-rotation-fns.test.ts b/backend/src/ee/services/secret-rotation-v2/azure-client-secret/azure-client-secret-rotation-fns.test.tsnew file mode 100644index 0000000000..5591bd5380--- /dev/null+++ b/backend/src/ee/services/secret-rotation-v2/azure-client-secret/azure-client-secret-rotation-fns.test.ts@@ -0,0 +1,191 @@+/* eslint-disable @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-argument */+import { AxiosError } from "axios";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";++// `vi.mock` factories are hoisted above imports, so the spies they reference must come from `vi.hoisted`.+const { getMock, postMock } = vi.hoisted(() => ({+ getMock: vi.fn(),+ postMock: vi.fn()+}));++vi.mock("@app/lib/config/request", () => ({+ request: {+ get: (...args: unknown[]) => (getMock as any)(...args),+ post: (...args: unknown[]) => (postMock as any)(...args)+ }+}));+vi.mock("@app/lib/logger", () => ({+ logger: { error: vi.fn(), info: vi.fn(), warn: vi.fn(), debug: vi.fn() }+}));+vi.mock("@app/services/app-connection/azure-client-secrets/azure-client-secrets-connection-fns", () => ({+ getAzureConnectionAccessToken: vi.fn(async () => "access-token")+}));++// eslint-disable-next-line import/first+import { azureClientSecretRotationFactory } from "./azure-client-secret-rotation-fns";++const OBJECT_ID = "app-object-id";+const OLD_KEY_ID = "old-key-id";+const NEW_KEY_ID = "new-key-id";++const concurrentRequestError = () =>+ new AxiosError("Request failed with status code 400", "ERR_BAD_REQUEST", undefined, undefined, {+ status: 400,+ statusText: "Bad Request",+ headers: {},+ config: {} as any,+ data: {+ error: {+ code: "Request_BadRequest",+ message: "Error due to concurrent requests being made to the tenant. Please wait briefly and retry."+ }+ }+ });++const forbiddenError = () =>+ new AxiosError("Request failed with status code 403", "ERR_BAD_REQUEST", undefined, undefined, {+ status: 403,+ statusText: "Forbidden",+ headers: {},+ config: {} as any,+ data: {+ error: { code: "Authorization_RequestDenied", message: "Insufficient privileges to complete the operation." }+ }+ });++const isRemovePassword = (url: string) => url.endsWith("/removePassword");++const removePasswordCalls = (keyId?: string) =>+ postMock.mock.calls.filter(([url, body]) => isRemovePassword(url) && (keyId === undefined || body.keyId === keyId));++const addPasswordCalls = () => postMock.mock.calls.filter(([url]) => url.endsWith("/addPassword"));++// removePassword behaviour is keyed by keyId so a test can fail the old secret's revoke while letting cleanup succeed.+const mockGraph = (removePassword: Record<string, () => Promise<unknown>>) => {+ getMock.mockResolvedValue({ data: { value: [{ keyId: OLD_KEY_ID }, { keyId: NEW_KEY_ID }] } });+ postMock.mockImplementation(async (url: string, body: any) => {+ if (url.endsWith("/addPassword")) return { data: { secretText: "new-secret", keyId: NEW_KEY_ID } };+ if (isRemovePassword(url)) return removePassword[body.keyId]();+ throw new Error(`unexpected request to ${url}`);+ });+};++const failTimesThenSucceed = (failures: number) => {+ let remaining = failures;+ return async () => {+ if (remaining > 0) {+ remaining -= 1;+ throw concurrentRequestError();+ }+ return { data: {} };+ };+};++const alwaysFail = (error: () => Error) => async () => {+ throw error();+};++const succeed = async () => ({ data: {} });++const makeFactory = () =>+ azureClientSecretRotationFactory(+ {+ connection: { id: "connection-id", credentials: { tenantId: "tenant-id" } },+ parameters: { objectId: OBJECT_ID, clientId: "client-id" },+ secretsMapping: { clientSecret: "AZURE_CLIENT_SECRET", clientId: "AZURE_CLIENT_ID" },+ rotationInterval: 30+ } as any,+ {} as any,+ {} as any,+ {} as any,+ {} as any,+ {} as any+ );++// The backoff sleeps are real timers, so the rotation is driven to completion under fake timers.+const rotate = async (callback: (credentials: unknown) => Promise<unknown>) => {+ const rotation = makeFactory().rotateCredentials(+ { keyId: OLD_KEY_ID, clientId: "client-id", clientSecret: "old-secret" } as any,+ callback as any,+ {} as any+ );+ // Attach the handler before advancing timers so a rejection is never left unobserved.+ const settled = rotation.then(+ (value) => ({ status: "fulfilled" as const, value }),+ (reason: unknown) => ({ status: "rejected" as const, reason })+ );+ await vi.runAllTimersAsync();+ return settled;+};++describe("azureClientSecretRotationFactory.rotateCredentials", () => {+ beforeEach(() => {+ vi.useFakeTimers();+ getMock.mockReset();+ postMock.mockReset();+ });++ afterEach(() => {+ vi.useRealTimers();+ });++ it("retries the revoke on Graph's concurrent request error and persists the new secret once it succeeds", async () => {+ mockGraph({ [OLD_KEY_ID]: failTimesThenSucceed(2) });+ const callback = vi.fn(async (credentials: unknown) => credentials);++ const result = await rotate(callback);++ expect(result.status).toBe("fulfilled");+ expect(addPasswordCalls()).toHaveLength(1);+ expect(removePasswordCalls(OLD_KEY_ID)).toHaveLength(3);+ expect(removePasswordCalls(NEW_KEY_ID)).toHaveLength(0);+ expect(callback).toHaveBeenCalledTimes(1);+ expect(callback).toHaveBeenCalledWith({ clientSecret: "new-secret", keyId: NEW_KEY_ID, clientId: "client-id" });+ });++ it("removes the newly created secret and fails the rotation when the revoke keeps failing", async () => {+ mockGraph({ [OLD_KEY_ID]: alwaysFail(concurrentRequestError), [NEW_KEY_ID]: succeed });+ const callback = vi.fn(async (credentials: unknown) => credentials);++ const result = await rotate(callback);++ expect(result.status).toBe("rejected");+ expect((result as { reason: Error }).reason.message).toContain(+ `Failed to remove client secret with keyId ${OLD_KEY_ID} from app ${OBJECT_ID}`+ );+ expect((result as { reason: Error }).reason.message).not.toContain("removed manually");+ expect(addPasswordCalls()).toHaveLength(1);+ expect(removePasswordCalls(OLD_KEY_ID)).toHaveLength(4);+ expect(removePasswordCalls(NEW_KEY_ID)).toHaveLength(1);+ expect(callback).not.toHaveBeenCalled();+ });++ it("reports the orphaned new secret when the cleanup fails too", async () => {+ mockGraph({ [OLD_KEY_ID]: alwaysFail(concurrentRequestError), [NEW_KEY_ID]: alwaysFail(concurrentRequestError) });+ const callback = vi.fn(async (credentials: unknown) => credentials);++ const result = await rotate(callback);++ expect(result.status).toBe("rejected");+ const { message } = (result as { reason: Error }).reason;+ expect(message).toContain(`Failed to remove client secret with keyId ${OLD_KEY_ID} from app ${OBJECT_ID}`);+ expect(message).toContain(`keyId ${NEW_KEY_ID} could not be cleaned up from app ${OBJECT_ID}`);+ expect(message).toContain("removed manually");+ expect(removePasswordCalls(OLD_KEY_ID)).toHaveLength(4);+ expect(removePasswordCalls(NEW_KEY_ID)).toHaveLength(4);+ expect(callback).not.toHaveBeenCalled();+ });++ it("does not retry revoke failures other than the concurrent request error", async () => {+ mockGraph({ [OLD_KEY_ID]: alwaysFail(forbiddenError), [NEW_KEY_ID]: succeed });+ const callback = vi.fn(async (credentials: unknown) => credentials);++ const result = await rotate(callback);++ expect(result.status).toBe("rejected");+ expect((result as { reason: Error }).reason.message).toContain("Insufficient privileges");+ expect(removePasswordCalls(OLD_KEY_ID)).toHaveLength(1);+ expect(removePasswordCalls(NEW_KEY_ID)).toHaveLength(1);+ expect(callback).not.toHaveBeenCalled();+ });+});diff --git a/backend/src/ee/services/secret-rotation-v2/azure-client-secret/azure-client-secret-rotation-fns.ts b/backend/src/ee/services/secret-rotation-v2/azure-client-secret/azure-client-secret-rotation-fns.tsindex 0e9c4567b4..d55a2de65f 100644--- a/backend/src/ee/services/secret-rotation-v2/azure-client-secret/azure-client-secret-rotation-fns.ts+++ b/backend/src/ee/services/secret-rotation-v2/azure-client-secret/azure-client-secret-rotation-fns.ts@@ -16,6 +16,7 @@ import {} from "@app/ee/services/secret-rotation-v2/secret-rotation-v2-types";import { request } from "@app/lib/config/request";import { BadRequestError } from "@app/lib/errors";+import { logger } from "@app/lib/logger";import { getAzureConnectionAccessToken } from "@app/services/app-connection/azure-client-secrets/azure-client-secrets-connection-fns";const GRAPH_API_BASE = "https://graph.microsoft.com/v1.0";@@ -23,12 +24,40 @@ const GRAPH_API_BASE = "https://graph.microsoft.com/v1.0";type AzureErrorResponse = { error: { message: string } };const EXPIRY_PADDING_IN_DAYS = 3;+const AZURE_CONCURRENT_REQUEST_MAX_RETRIES = 3;+const AZURE_CONCURRENT_REQUEST_BASE_DELAY_MS = 2000;-const sleep = async () =>- new Promise((resolve) => {- setTimeout(resolve, 1000);+const sleep = async (ms = 1000) =>+ new Promise<void>((resolve) => {+ setTimeout(resolve, ms);});+const getAzureErrorMessage = (error: AxiosError): string | undefined => {+ const message = (error.response?.data as Partial<AzureErrorResponse> | undefined)?.error?.message;+ return typeof message === "string" ? message : undefined;+};++const getErrorMessage = (error: unknown): string => (error instanceof Error ? error.message : "Unknown error");++// Graph rejects back-to-back writes to the same app registration with a transient "concurrent requests" error+const isAzureConcurrentRequestError = (error: unknown): boolean =>+ error instanceof AxiosError && (getAzureErrorMessage(error) ?? "").toLowerCase().includes("concurrent requests");++const withAzureConcurrentRequestRetry = async <T>(fn: () => Promise<T>, context: string, attempt = 0): Promise<T> => {+ try {+ return await fn();+ } catch (error: unknown) {+ if (!isAzureConcurrentRequestError(error) || attempt >= AZURE_CONCURRENT_REQUEST_MAX_RETRIES) throw error;++ const delay = AZURE_CONCURRENT_REQUEST_BASE_DELAY_MS * 2 ** attempt;+ logger.info(+ `secretRotation: Azure concurrent request error on ${context}, retrying in ${delay}ms (attempt ${attempt + 1}/${AZURE_CONCURRENT_REQUEST_MAX_RETRIES})`+ );+ await sleep(delay);+ return withAzureConcurrentRequestRetry(fn, context, attempt + 1);+ }+};+export const azureClientSecretRotationFactory: TRotationFactory<TAzureClientSecretRotationWithConnection,TAzureClientSecretRotationGeneratedCredentials@@ -84,15 +113,7 @@ export const azureClientSecretRotationFactory: TRotationFactory<};} catch (error: unknown) {if (error instanceof AxiosError) {- let message;- if (- error.response?.data &&- typeof error.response.data === "object" &&- "error" in error.response.data &&- typeof (error.response.data as AzureErrorResponse).error.message === "string"- ) {- message = (error.response.data as AzureErrorResponse).error.message;- }+ const message = getAzureErrorMessage(error);throw new BadRequestError({message: `Failed to add client secret to Azure app ${objectId}: ${message || error.message || "Unknown error"@@ -113,25 +134,21 @@ export const azureClientSecretRotationFactory: TRotationFactory<const endpoint = `${GRAPH_API_BASE}/applications/${objectId}/passwordCredentials`;try {- const { data } = await request.get<{ value: Array<{ keyId: string }> }>(endpoint, {- headers: {- Authorization: `Bearer ${accessToken}`,- "Content-Type": "application/json"- }- });+ const { data } = await withAzureConcurrentRequestRetry(+ () =>+ request.get<{ value: Array<{ keyId: string }> }>(endpoint, {+ headers: {+ Authorization: `Bearer ${accessToken}`,+ "Content-Type": "application/json"+ }+ }),+ "list credentials"+ );return data.value?.some((credential) => credential.keyId === keyId) || false;} catch (error: unknown) {if (error instanceof AxiosError) {- let message;- if (- error.response?.data &&- typeof error.response.data === "object" &&- "error" in error.response.data &&- typeof (error.response.data as AzureErrorResponse).error.message === "string"- ) {- message = (error.response.data as AzureErrorResponse).error.message;- }+ const message = getAzureErrorMessage(error);throw new BadRequestError({message: `Failed to check credential existence for app ${objectId}: ${message || error.message || "Unknown error"@@ -144,42 +161,28 @@ export const azureClientSecretRotationFactory: TRotationFactory<}};- /**- * Revokes a client secret from the Azure app using its keyId.- * First checks if the credential exists before attempting revocation.- */- const revokeCredential = async (keyId: string) => {- // Check if credential exists before attempting revocation- const exists = await credentialExists(keyId);- if (!exists) {- return; // Credential doesn't exist, nothing to revoke- }-+ const $removeClientSecret = async (keyId: string) => {const accessToken = await getAzureConnectionAccessToken(connection.id, appConnectionDAL, kmsService);const endpoint = `${GRAPH_API_BASE}/applications/${objectId}/removePassword`;try {- await request.post(- endpoint,- { keyId },- {- headers: {- Authorization: `Bearer ${accessToken}`,- "Content-Type": "application/json"- }- }+ await withAzureConcurrentRequestRetry(+ () =>+ request.post(+ endpoint,+ { keyId },+ {+ headers: {+ Authorization: `Bearer ${accessToken}`,+ "Content-Type": "application/json"+ }+ }+ ),+ "remove credential");} catch (error: unknown) {if (error instanceof AxiosError) {- let message;- if (- error.response?.data &&- typeof error.response.data === "object" &&- "error" in error.response.data &&- typeof (error.response.data as AzureErrorResponse).error.message === "string"- ) {- message = (error.response.data as AzureErrorResponse).error.message;- }+ const message = getAzureErrorMessage(error);throw new BadRequestError({message: `Failed to remove client secret with keyId ${keyId} from app ${objectId}: ${message || error.message || "Unknown error"@@ -192,6 +195,19 @@ export const azureClientSecretRotationFactory: TRotationFactory<}};+ /**+ * Revokes a client secret from the Azure app using its keyId.+ * First checks if the credential exists before attempting revocation.+ */+ const revokeCredential = async (keyId: string) => {+ const exists = await credentialExists(keyId);+ if (!exists) {+ return; // Credential doesn't exist, nothing to revoke+ }++ await $removeClientSecret(keyId);+ };+/*** Issues a new set of credentials.*/@@ -233,7 +249,21 @@ export const azureClientSecretRotationFactory: TRotationFactory<) => {const newCredentials = await $rotateClientSecret();if (oldCredentials?.keyId) {- await revokeCredential(oldCredentials.keyId);+ try {+ await revokeCredential(oldCredentials.keyId);+ } catch (revokeError: unknown) {+ // only two credentials are tracked, so persisting the new secret now would drop the old keyId from tracking+ try {+ await sleep(AZURE_CONCURRENT_REQUEST_BASE_DELAY_MS);+ await $removeClientSecret(newCredentials.keyId);+ } catch (cleanupError: unknown) {+ const cleanupMessage = `The newly created client secret with keyId ${newCredentials.keyId} could not be cleaned up from app ${objectId} and may need to be removed manually`;+ throw new BadRequestError({+ message: `${getErrorMessage(revokeError)} ${cleanupMessage}: ${getErrorMessage(cleanupError)}`+ });+ }+ throw revokeError;+ }}return callback(newCredentials);