Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.
TypeScript acmeagentsai-agentscertificate-managementdevsecopsenvironment-variableskubernetesopen-sourcepampkipostgresprivate-caprivileged-access-managementsecret-managementsecret-managersecretssecrets-managementsecuritysecurity-toolsvault
feat(telemetry): enhance dynamic secret revocation metrics and add new key store functionality
Victor Hugo dos Santosccdd1aa5976c
diff --git a/backend/e2e-test/mocks/keystore.ts b/backend/e2e-test/mocks/keystore.tsindex e68c7e26a0..9ae56053b1 100644--- a/backend/e2e-test/mocks/keystore.ts+++ b/backend/e2e-test/mocks/keystore.ts@@ -30,6 +30,11 @@ export const mockKeyStore = (): TKeyStoreFactory => {store[key] = value;return "OK";},+ setItemWithExpiryReturningPrevious: async (key, _expiryInSeconds, value) => {+ const previous = store[key];+ store[key] = value;+ return previous === undefined ? null : previous.toString();+ },deleteItem: async (key) => {delete store[key];delete hashStore[key];diff --git a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-queue.test.ts b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-queue.test.tsnew file mode 100644index 0000000000..7c4665f664--- /dev/null+++ b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-queue.test.ts@@ -0,0 +1,333 @@+import { beforeEach, describe, expect, test, vi } from "vitest";++import { KeyStorePrefixes } from "@app/keystore/keystore";+import { inMemoryKeyStore } from "@app/keystore/memory";+import { logger } from "@app/lib/logger";+import {+ isTelemetryEnabled,+ recordDynamicSecretOrphanedLeaseMetric,+ recordDynamicSecretRevocationOutcomeMetric,+ recordDynamicSecretStatusChangeMetric+} from "@app/lib/telemetry/metrics";+import { QueueJobs, QueueName } from "@app/queue";++import { DynamicSecretStatus } from "../dynamic-secret/dynamic-secret-types";+import { DynamicSecretProviders } from "../dynamic-secret/providers/models";+import { dynamicSecretLeaseQueueServiceFactory } from "./dynamic-secret-lease-queue";++vi.mock("@app/lib/config/env", async (importOriginal) => ({+ ...(await importOriginal<typeof import("@app/lib/config/env")>()),+ getConfig: () => ({ isDevelopmentMode: false })+}));++vi.mock("@app/lib/logger", () => ({+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }+}));++vi.mock("@app/lib/telemetry/metrics", async (importOriginal) => ({+ ...(await importOriginal<typeof import("@app/lib/telemetry/metrics")>()),+ isTelemetryEnabled: vi.fn(() => true),+ recordDynamicSecretOrphanedLeaseMetric: vi.fn(),+ recordDynamicSecretRevocationOutcomeMetric: vi.fn(),+ recordDynamicSecretStatusChangeMetric: vi.fn()+}));++const PROVIDER = DynamicSecretProviders.SqlDatabase;+const CONFIG_ID = "dynamic-secret-1";+const MAX_ATTEMPTS = 3;++const dynamicSecretCfg = {+ id: CONFIG_ID,+ type: PROVIDER,+ folderId: "folder-1",+ encryptedInput: Buffer.from("{}"),+ status: DynamicSecretStatus.Deleting+};++const lease = (id: string) => ({+ id,+ dynamicSecretId: CONFIG_ID,+ externalEntityId: `entity-${id}`,+ config: {},+ encryptedLeaseData: null,+ dynamicSecret: dynamicSecretCfg+});++type THandler = (job: unknown) => Promise<void>;++const createHarness = () => {+ const handlers = new Map<string, THandler>();+ const queueService = {+ start: vi.fn((name: string, fn: THandler) => {+ handlers.set(name, fn);+ }),+ queue: vi.fn(async () => undefined),+ stopJobById: vi.fn(async () => undefined)+ };++ const leases = new Map<string, ReturnType<typeof lease>>();+ const dynamicSecretLeaseDAL = {+ findById: vi.fn(async (id: string) => leases.get(id)),+ deleteById: vi.fn(async (id: string) => {+ leases.delete(id);+ }),+ find: vi.fn(async () => [...leases.values()]),+ updateById: vi.fn(async () => ({}))+ };+ const cfg = { ...dynamicSecretCfg };+ const dynamicSecretDAL = {+ findById: vi.fn(async () => cfg),+ deleteById: vi.fn(async () => cfg),+ updateById: vi.fn(async (_id: string, data: Partial<typeof cfg>) => Object.assign(cfg, data))+ };+ const revoke = vi.fn<(inputs: unknown, entityId: string) => Promise<{ entityId: string }>>(+ async (_inputs, entityId) => ({ entityId })+ );+ const keyStore = inMemoryKeyStore();++ dynamicSecretLeaseQueueServiceFactory({+ queueService: queueService as never,+ dynamicSecretLeaseDAL: dynamicSecretLeaseDAL as never,+ dynamicSecretDAL: dynamicSecretDAL as never,+ dynamicSecretProviders: { [PROVIDER]: { revoke } } as never,+ folderDAL: { findById: vi.fn(async () => ({ id: "folder-1", projectId: "project-1" })) } as never,+ kmsService: {+ createCipherPairWithDataKey: vi.fn(async () => ({ decryptor: () => Buffer.from("{}") }))+ } as never,+ keyStore,+ smtpService: {} as never,+ userDAL: {} as never,+ identityDAL: {} as never,+ projectMembershipDAL: {} as never,+ projectDAL: {} as never+ });++ const run = (job: { name: string; id: string; data: unknown; attempt?: number; attempts?: number }) =>+ handlers.get(QueueName.DynamicSecretRevocation)!({+ name: job.name,+ id: job.id,+ data: job.data,+ attemptsMade: (job.attempt ?? 1) - 1,+ opts: { attempts: job.attempts ?? MAX_ATTEMPTS }+ });++ const runScheduled = (leaseId: string, attempt = 1) =>+ run({+ name: QueueJobs.DynamicSecretRevocation,+ id: leaseId,+ data: { leaseId, dynamicSecretId: CONFIG_ID },+ attempt+ });++ const runRetry = (leaseId: string, attempt = 1) =>+ run({+ name: QueueJobs.DynamicSecretRevocation,+ id: `${leaseId}-retry`,+ data: { leaseId, dynamicSecretId: CONFIG_ID, isRetry: true },+ attempt+ });++ const runPruning = (attempt = 1) =>+ run({+ name: QueueJobs.DynamicSecretPruning,+ id: `dynamic-secret-lease-pruning-${CONFIG_ID}`,+ data: { dynamicSecretCfgId: CONFIG_ID },+ attempt+ });++ const addLease = (id: string) => leases.set(id, lease(id));++ return { queueService, revoke, keyStore, addLease, runScheduled, runRetry, runPruning };+};++const outcomeCalls = () => vi.mocked(recordDynamicSecretRevocationOutcomeMetric).mock.calls.map(([params]) => params);+const statusChangeCalls = () => vi.mocked(recordDynamicSecretStatusChangeMetric).mock.calls.map(([params]) => params);+const orphanedCalls = () => vi.mocked(recordDynamicSecretOrphanedLeaseMetric).mock.calls.map(([params]) => params);+const retryQueued = (queue: ReturnType<typeof createHarness>["queueService"]["queue"]) =>+ queue.mock.calls.filter((call) => (call as unknown[])[1] === QueueJobs.DynamicSecretRevocation);++const failingKey = KeyStorePrefixes.DynamicSecretRevocationFailing(CONFIG_ID);++describe("dynamic secret lease revocation metrics", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ vi.mocked(isTelemetryEnabled).mockReturnValue(true);+ });++ describe("scheduled phase", () => {+ test("records one success and no status change for a healthy config", async () => {+ const harness = createHarness();+ harness.addLease("lease-1");++ await harness.runScheduled("lease-1");++ expect(outcomeCalls()).toEqual([+ { provider: PROVIDER, phase: "scheduled", outcome: "success", attemptsExhausted: false }+ ]);+ expect(statusChangeCalls()).toEqual([]);+ });++ test("records exactly one failure per lease, on the final scheduled attempt", async () => {+ const harness = createHarness();+ harness.addLease("lease-1");+ harness.revoke.mockRejectedValue(new Error("connection refused"));++ await expect(harness.runScheduled("lease-1", 1)).rejects.toThrow("connection refused");+ await expect(harness.runScheduled("lease-1", 2)).rejects.toThrow("connection refused");+ expect(outcomeCalls()).toEqual([]);+ // Handing off before the last attempt would let the retry job race the scheduled retries.+ expect(retryQueued(harness.queueService.queue)).toHaveLength(0);++ await expect(harness.runScheduled("lease-1", 3)).rejects.toThrow("connection refused");+ expect(outcomeCalls()).toEqual([+ { provider: PROVIDER, phase: "scheduled", outcome: "failure", attemptsExhausted: true }+ ]);+ expect(retryQueued(harness.queueService.queue)).toHaveLength(1);+ });++ test("records the success of a later scheduled attempt as the lease's only outcome", async () => {+ const harness = createHarness();+ harness.addLease("lease-1");+ harness.revoke.mockRejectedValueOnce(new Error("timeout"));++ await expect(harness.runScheduled("lease-1", 1)).rejects.toThrow("timeout");+ await harness.runScheduled("lease-1", 2);++ expect(outcomeCalls()).toEqual([+ { provider: PROVIDER, phase: "scheduled", outcome: "success", attemptsExhausted: false }+ ]);+ });++ test("flips to failed on a config's first failed lease and not on the next", async () => {+ const harness = createHarness();+ harness.addLease("lease-1");+ harness.addLease("lease-2");+ harness.revoke.mockRejectedValue(new Error("connection refused"));++ await expect(harness.runScheduled("lease-1", MAX_ATTEMPTS)).rejects.toThrow();+ await expect(harness.runScheduled("lease-2", MAX_ATTEMPTS)).rejects.toThrow();++ expect(statusChangeCalls()).toEqual([{ provider: PROVIDER, to: "failed" }]);+ expect(await harness.keyStore.getItem(failingKey)).toBe("1");+ });++ test("flips to healthy once when a failing config revokes again", async () => {+ const harness = createHarness();+ harness.addLease("lease-1");+ harness.addLease("lease-2");+ harness.addLease("lease-3");+ harness.revoke.mockRejectedValueOnce(new Error("connection refused"));++ await expect(harness.runScheduled("lease-1", MAX_ATTEMPTS)).rejects.toThrow();+ await harness.runScheduled("lease-2");+ await harness.runScheduled("lease-3");++ expect(statusChangeCalls()).toEqual([+ { provider: PROVIDER, to: "failed" },+ { provider: PROVIDER, to: "healthy" }+ ]);+ expect(await harness.keyStore.getItem(failingKey)).toBeNull();+ });++ test("does not let a Redis error fail or change the revocation", async () => {+ const harness = createHarness();+ harness.addLease("lease-1");+ harness.addLease("lease-2");+ const redisError = new Error("redis unavailable");+ vi.spyOn(harness.keyStore, "getItem").mockRejectedValue(redisError);+ vi.spyOn(harness.keyStore, "setItemWithExpiryReturningPrevious").mockRejectedValue(redisError);++ await expect(harness.runScheduled("lease-1")).resolves.toBeUndefined();++ harness.revoke.mockRejectedValueOnce(new Error("connection refused"));+ await expect(harness.runScheduled("lease-2", MAX_ATTEMPTS)).rejects.toThrow("connection refused");++ expect(statusChangeCalls()).toEqual([]);+ expect(outcomeCalls().map(({ outcome }) => outcome)).toEqual(["success", "failure"]);+ expect(retryQueued(harness.queueService.queue)).toHaveLength(1);+ expect(vi.mocked(logger.warn)).toHaveBeenCalledTimes(2);+ });++ test("skips the status change entirely while telemetry is disabled", async () => {+ vi.mocked(isTelemetryEnabled).mockReturnValue(false);+ const harness = createHarness();+ harness.addLease("lease-1");+ const setPrevious = vi.spyOn(harness.keyStore, "setItemWithExpiryReturningPrevious");+ harness.revoke.mockRejectedValue(new Error("connection refused"));++ await expect(harness.runScheduled("lease-1", MAX_ATTEMPTS)).rejects.toThrow();++ expect(setPrevious).not.toHaveBeenCalled();+ });++ test("records nothing when the lease no longer exists", async () => {+ const harness = createHarness();++ await harness.runScheduled("missing-lease");++ expect(outcomeCalls()).toEqual([]);+ expect(statusChangeCalls()).toEqual([]);+ });+ });++ describe("retry phase", () => {+ test("records every attempt, exhausted only on the last, and orphans the lease there", async () => {+ const harness = createHarness();+ harness.addLease("lease-1");+ harness.revoke.mockRejectedValue(new Error("connection refused"));++ for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt += 1) {+ // eslint-disable-next-line no-await-in-loop+ await expect(harness.runRetry("lease-1", attempt)).rejects.toThrow();+ }++ expect(outcomeCalls()).toEqual([+ { provider: PROVIDER, phase: "retry", outcome: "failure", attemptsExhausted: false },+ { provider: PROVIDER, phase: "retry", outcome: "failure", attemptsExhausted: false },+ { provider: PROVIDER, phase: "retry", outcome: "failure", attemptsExhausted: true }+ ]);+ expect(orphanedCalls()).toEqual([{ provider: PROVIDER }]);+ expect(statusChangeCalls()).toEqual([]);+ });+ });++ describe("pruning", () => {+ test("records each lease once and orphans only the ones left unrevoked", async () => {+ const harness = createHarness();+ harness.addLease("lease-1");+ harness.addLease("lease-2");+ harness.addLease("lease-3");+ harness.revoke.mockImplementation(async (_inputs, entityId) => {+ if (entityId === "entity-lease-2") throw new Error("permission denied");+ return { entityId };+ });++ await expect(harness.runPruning(1)).rejects.toThrow("permission denied");+ // The config is now FailedDeletion, so BullMQ's remaining attempts return without revoking.+ await harness.runPruning(2);+ await harness.runPruning(3);++ expect(outcomeCalls()).toEqual([+ { provider: PROVIDER, phase: "pruning", outcome: "success", attemptsExhausted: true },+ { provider: PROVIDER, phase: "pruning", outcome: "failure", attemptsExhausted: true },+ { provider: PROVIDER, phase: "pruning", outcome: "success", attemptsExhausted: true }+ ]);+ expect(orphanedCalls()).toEqual([{ provider: PROVIDER, count: 1 }]);+ expect(harness.revoke).toHaveBeenCalledTimes(3);+ });++ test("records a success per lease when every revoke succeeds", async () => {+ const harness = createHarness();+ harness.addLease("lease-1");+ harness.addLease("lease-2");++ await harness.runPruning(1);++ expect(outcomeCalls()).toEqual([+ { provider: PROVIDER, phase: "pruning", outcome: "success", attemptsExhausted: false },+ { provider: PROVIDER, phase: "pruning", outcome: "success", attemptsExhausted: false }+ ]);+ expect(orphanedCalls()).toEqual([]);+ });+ });+});diff --git a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-queue.ts b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-queue.tsindex b86519d27c..b653314668 100644--- a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-queue.ts+++ b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-queue.ts@@ -1,9 +1,15 @@import { ProjectMembershipRole } from "@app/db/schemas";+import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";import { getConfig } from "@app/lib/config/env";import { applyJitter } from "@app/lib/delay";import { DisableRotationErrors, NotFoundError } from "@app/lib/errors";import { logger } from "@app/lib/logger";-import { recordDynamicSecretOrphanedLeaseMetric } from "@app/lib/telemetry/metrics";+import {+ isTelemetryEnabled,+ recordDynamicSecretOrphanedLeaseMetric,+ recordDynamicSecretRevocationOutcomeMetric,+ recordDynamicSecretStatusChangeMetric+} from "@app/lib/telemetry/metrics";import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";import { TIdentityDALFactory } from "@app/services/identity/identity-dal";import { TKmsServiceFactory } from "@app/services/kms/kms-service";@@ -33,6 +39,7 @@ type TDynamicSecretLeaseQueueServiceFactoryDep = {dynamicSecretProviders: Record<DynamicSecretProviders, TDynamicProviderFns>;kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;folderDAL: Pick<TSecretFolderDALFactory, "findById" | "findSecretPathByFolderIds">;+ keyStore: Pick<TKeyStoreFactory, "getItem" | "deleteItem" | "setItemWithExpiryReturningPrevious">;};export type TDynamicSecretLeaseQueueServiceFactory = {@@ -43,6 +50,7 @@ export type TDynamicSecretLeaseQueueServiceFactory = {};const MAX_REVOCATION_RETRY_COUNT = 3;+const REVOCATION_FAILING_STATUS_EXPIRY_SECONDS = 7 * 24 * 60 * 60;export const dynamicSecretLeaseQueueServiceFactory = ({queueService,@@ -53,7 +61,8 @@ export const dynamicSecretLeaseQueueServiceFactory = ({folderDAL,projectMembershipDAL,projectDAL,- smtpService+ smtpService,+ keyStore}: TDynamicSecretLeaseQueueServiceFactoryDep): TDynamicSecretLeaseQueueServiceFactory => {const pruneDynamicSecret = async (dynamicSecretCfgId: string) => {await queueService.queue(@@ -137,16 +146,69 @@ export const dynamicSecretLeaseQueueServiceFactory = ({);};+ // Dynamic secret configs have no revocation status column, so whether a config's scheduled revocations are+ // failing is kept in Redis. This only feeds a metric and must never fail or retry the revocation job.+ const $recordScheduledRevocationStatusChange = async ({+ dynamicSecretId,+ provider,+ isRevoked+ }: {+ dynamicSecretId: string;+ provider: string;+ isRevoked: boolean;+ }) => {+ if (!isTelemetryEnabled()) return;++ const key = KeyStorePrefixes.DynamicSecretRevocationFailing(dynamicSecretId);+ try {+ if (isRevoked) {+ if (!(await keyStore.getItem(key))) return;+ // Only the caller that actually deleted the key reports the recovery.+ const deleted = await keyStore.deleteItem(key);+ if (deleted > 0) recordDynamicSecretStatusChangeMetric({ provider, to: "healthy" });+ return;+ }++ const previous = await keyStore.setItemWithExpiryReturningPrevious(+ key,+ REVOCATION_FAILING_STATUS_EXPIRY_SECONDS,+ "1"+ );+ if (!previous) recordDynamicSecretStatusChangeMetric({ provider, to: "failed" });+ } catch (error) {+ logger.warn(+ error,+ `Failed to record dynamic secret revocation status change [dynamicSecretId=${dynamicSecretId}] [provider=${provider}]`+ );+ }+ };++ const $recordPruningOutcomes = (provider: string, revoked: boolean[], attemptsExhausted: boolean) => {+ revoked.forEach((isRevoked) =>+ recordDynamicSecretRevocationOutcomeMetric({+ provider,+ phase: "pruning",+ outcome: isRevoked ? "success" : "failure",+ attemptsExhausted+ })+ );+ };+const $dynamicSecretQueueJob = async (jobName: string,jobId: string,data: { leaseId: string; dynamicSecretId: string; isRetry?: boolean } | { dynamicSecretCfgId: string },- retryCount?: number+ retryCount?: number,+ maxAttempts = 1): Promise<void> => {let providerType: string | undefined;+ let configId: string | undefined;+ const isFinalAttempt = retryCount !== undefined && retryCount >= maxAttempts;+ // Per-lease revoke results of this pruning run, so a failed run reports each lease once.+ let pruningRevoked: boolean[] | undefined;try {if (jobName === QueueJobs.DynamicSecretRevocation) {- const { leaseId } = data as { leaseId: string };+ const { leaseId, isRetry } = data as { leaseId: string; isRetry?: boolean };logger.info("Dynamic secret lease revocation started: ", leaseId, jobId);const dynamicSecretLease = await dynamicSecretLeaseDAL.findById(leaseId);@@ -154,9 +216,13 @@ export const dynamicSecretLeaseQueueServiceFactory = ({return;}- const folder = await folderDAL.findById(dynamicSecretLease.dynamicSecret.folderId);+ const dynamicSecretCfg = dynamicSecretLease.dynamicSecret;+ providerType = dynamicSecretCfg.type;+ configId = dynamicSecretCfg.id;++ const folder = await folderDAL.findById(dynamicSecretCfg.folderId);if (!folder) {- logger.info(`Failed to find folder with ${dynamicSecretLease.dynamicSecret.folderId}`);+ logger.info(`Failed to find folder with ${dynamicSecretCfg.folderId}`);return;}@@ -165,8 +231,6 @@ export const dynamicSecretLeaseQueueServiceFactory = ({projectId: folder.projectId});- const dynamicSecretCfg = dynamicSecretLease.dynamicSecret;- providerType = dynamicSecretCfg.type;const selectedProvider = dynamicSecretProviders[dynamicSecretCfg.type as DynamicSecretProviders];const decryptedStoredInput = JSON.parse(secretManagerDecryptor({ cipherTextBlob: dynamicSecretCfg.encryptedInput }).toString()@@ -182,6 +246,20 @@ export const dynamicSecretLeaseQueueServiceFactory = ({dynamicSecretLease.config as TDynamicSecretLeaseConfig);await dynamicSecretLeaseDAL.deleteById(dynamicSecretLease.id);++ recordDynamicSecretRevocationOutcomeMetric({+ provider: providerType,+ phase: isRetry ? "retry" : "scheduled",+ outcome: "success",+ attemptsExhausted: isFinalAttempt+ });+ if (!isRetry) {+ await $recordScheduledRevocationStatusChange({+ dynamicSecretId: configId,+ provider: providerType,+ isRevoked: true+ });+ }return;}@@ -218,8 +296,8 @@ export const dynamicSecretLeaseQueueServiceFactory = ({) as object;await Promise.all(dynamicSecretLeases.map(({ id }) => unsetLeaseRevocation(id)));- await Promise.all(- dynamicSecretLeases.map(({ externalEntityId, config, encryptedLeaseData }) =>+ const revocations = await Promise.allSettled(+ dynamicSecretLeases.map(async ({ externalEntityId, config, encryptedLeaseData }) =>selectedProvider.revoke(decryptedStoredInput,externalEntityId,@@ -231,9 +309,17 @@ export const dynamicSecretLeaseQueueServiceFactory = ({)));+ pruningRevoked = revocations.map((revocation) => revocation.status === "fulfilled");++ const failedRevocation = revocations.find(+ (revocation): revocation is PromiseRejectedResult => revocation.status === "rejected"+ );+ if (failedRevocation) throw failedRevocation.reason;}await dynamicSecretDAL.deleteById(dynamicSecretCfgId);++ if (pruningRevoked) $recordPruningOutcomes(providerType, pruningRevoked, isFinalAttempt);}logger.info("Finished dynamic secret job", jobId);} catch (error) {@@ -245,8 +331,13 @@ export const dynamicSecretLeaseQueueServiceFactory = ({status: DynamicSecretStatus.FailedDeletion,statusDetails: (error as Error)?.message?.slice(0, 255)});- if (providerType) {- recordDynamicSecretOrphanedLeaseMetric({ provider: providerType });+ if (providerType && pruningRevoked && !(error instanceof DisableRotationErrors)) {+ $recordPruningOutcomes(providerType, pruningRevoked, true);++ const unrevokedLeaseCount = pruningRevoked.filter((isRevoked) => !isRevoked).length;+ if (unrevokedLeaseCount > 0) {+ recordDynamicSecretOrphanedLeaseMetric({ provider: providerType, count: unrevokedLeaseCount });+ }}}@@ -256,20 +347,42 @@ export const dynamicSecretLeaseQueueServiceFactory = ({isRetry?: boolean;dynamicSecretId: string;};+ const isRevocationFailure = !(error instanceof DisableRotationErrors);++ // A scheduled revocation reports one outcome per lease: the final attempt's, since BullMQ retries it.+ if (providerType && isRevocationFailure && (isRetry || isFinalAttempt)) {+ recordDynamicSecretRevocationOutcomeMetric({+ provider: providerType,+ phase: isRetry ? "retry" : "scheduled",+ outcome: "failure",+ attemptsExhausted: isFinalAttempt+ });+ if (!isRetry && configId) {+ await $recordScheduledRevocationStatusChange({+ dynamicSecretId: configId,+ provider: providerType,+ isRevoked: false+ });+ }+ }+await dynamicSecretLeaseDAL.updateById(leaseId, {status: DynamicSecretStatus.FailedDeletion,statusDetails: `${(error as Error)?.message?.slice(0, 255)} - Retrying automatically`});// only add to retry queue if this is not a retry, and if the error is not a DisableRotationErrors error- if (!isRetry && !(error instanceof DisableRotationErrors)) {- // if revocation fails, we should stop the job and queue a new job to retry the revocation at a later time.- await queueService.stopJobById(QueueName.DynamicSecretRevocation, jobId);- await queueFailedRevocation(leaseId, dynamicSecretId);+ if (!isRetry && isRevocationFailure) {+ // BullMQ retries the scheduled job (it cannot be removed while active), so hand off to the slower retry+ // queue only once its attempts are used up. Handing off earlier lets the retry job revoke the lease between+ // scheduled attempts, leaving the lease with no scheduled outcome.+ if (isFinalAttempt) {+ await queueFailedRevocation(leaseId, dynamicSecretId);+ }// if its the last attempt, and the error isn't a DisableRotationErrors error, send an email to the project admins (debounced)- } else if (isRetry && !(error instanceof DisableRotationErrors)) {- if (retryCount && retryCount === MAX_REVOCATION_RETRY_COUNT) {+ } else if (isRetry && isRevocationFailure) {+ if (isFinalAttempt) {// if all retries fail, we should also stop the automatic revocation job.// the ID of the revocation job is set to the leaseId, so we can use that to stop the job@@ -349,7 +462,7 @@ export const dynamicSecretLeaseQueueServiceFactory = ({};queueService.start(QueueName.DynamicSecretRevocation, async (job) => {- await $dynamicSecretQueueJob(job.name, job.id as string, job.data, job.attemptsMade + 1);+ await $dynamicSecretQueueJob(job.name, job.id as string, job.data, job.attemptsMade + 1, job.opts.attempts);});queueService.start(QueueName.DynamicSecretLeaseRevocationFailedEmail, async (job) => {diff --git a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-service.test.ts b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-service.test.tsnew file mode 100644index 0000000000..9058abfb98--- /dev/null+++ b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-service.test.ts@@ -0,0 +1,171 @@+import { createMongoAbility } from "@casl/ability";+import { beforeEach, describe, expect, test, vi } from "vitest";++import {+ recordDynamicSecretRevocationOutcomeMetric,+ recordDynamicSecretUnrecordedCredentialMetric+} from "@app/lib/telemetry/metrics";+import { ActorType, AuthMethod } from "@app/services/auth/auth-type";++import { DynamicSecretProviders } from "../dynamic-secret/providers/models";+import { dynamicSecretLeaseServiceFactory } from "./dynamic-secret-lease-service";++vi.mock("@app/lib/config/env", async (importOriginal) => ({+ ...(await importOriginal<typeof import("@app/lib/config/env")>()),+ getConfig: () => ({ MAX_LEASE_LIMIT: 100 })+}));++vi.mock("@app/lib/logger", () => ({+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }+}));++vi.mock("@app/lib/telemetry/metrics", async (importOriginal) => ({+ ...(await importOriginal<typeof import("@app/lib/telemetry/metrics")>()),+ recordDynamicSecretRevocationOutcomeMetric: vi.fn(),+ recordDynamicSecretUnrecordedCredentialMetric: vi.fn()+}));++const PROVIDER = DynamicSecretProviders.SqlDatabase;++const dynamicSecretCfg = {+ id: "dynamic-secret-1",+ type: PROVIDER,+ folderId: "folder-1",+ encryptedInput: Buffer.from("{}"),+ defaultTTL: "1h",+ maxTTL: null,+ usernameTemplate: null,+ metadata: []+};++const createService = () => {+ const provider = {+ create: vi.fn(async () => ({ entityId: "entity-1", data: {} })),+ revoke: vi.fn(async () => ({ entityId: "entity-1" }))+ };+ const dynamicSecretLeaseDAL = {+ countLeasesForDynamicSecret: vi.fn(async () => 0),+ create: vi.fn(async () => ({ id: "lease-1" })),+ findById: vi.fn(async () => ({+ id: "lease-1",+ dynamicSecretId: dynamicSecretCfg.id,+ externalEntityId: "entity-1",+ config: {},+ encryptedLeaseData: null,+ dynamicSecret: dynamicSecretCfg+ })),+ updateById: vi.fn(async () => ({})),+ deleteById: vi.fn(async () => ({}))+ };++ const service = dynamicSecretLeaseServiceFactory({+ dynamicSecretLeaseDAL: dynamicSecretLeaseDAL as never,+ dynamicSecretDAL: { findOne: vi.fn(async () => dynamicSecretCfg) } as never,+ dynamicSecretProviders: { [PROVIDER]: provider } as never,+ dynamicSecretQueueService: {+ setLeaseRevocation: vi.fn(async () => undefined),+ unsetLeaseRevocation: vi.fn(async () => undefined),+ queueFailedRevocation: vi.fn(async () => undefined)+ } as never,+ licenseService: { getPlan: vi.fn(async () => ({ dynamicSecret: true })) } as never,+ folderDAL: { findBySecretPath: vi.fn(async () => ({ id: "folder-1", envId: "env-1" })) } as never,+ permissionService: {+ getProjectPermission: vi.fn(async () => ({+ permission: createMongoAbility([{ action: "manage", subject: "all" }])+ }))+ } as never,+ projectDAL: { findProjectBySlug: vi.fn(async () => ({ id: "project-1" })) } as never,+ kmsService: {+ createCipherPairWithDataKey: vi.fn(async () => ({+ encryptor: () => ({ cipherTextBlob: Buffer.from("") }),+ decryptor: () => Buffer.from("{}")+ }))+ } as never,+ userDAL: { findById: vi.fn(async () => undefined) } as never,+ identityDAL: { findById: vi.fn(async () => undefined) } as never,+ secretValidationRuleService: { findConstraintsForGeneratedSecret: vi.fn(async () => ({ ruleNames: [] })) } as never+ });++ return { service, provider, dynamicSecretLeaseDAL };+};++const actor = {+ actor: ActorType.USER,+ actorId: "user-1",+ actorOrgId: "org-1",+ actorAuthMethod: AuthMethod.EMAIL,+ projectSlug: "project",+ environmentSlug: "dev",+ path: "/"+};++const outcomeCalls = () => vi.mocked(recordDynamicSecretRevocationOutcomeMetric).mock.calls.map(([params]) => params);++describe("dynamic secret lease service metrics", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ });++ describe("manual revocation", () => {+ test("records a success", async () => {+ const { service } = createService();++ await service.revokeLease({ ...actor, leaseId: "lease-1" });++ expect(outcomeCalls()).toEqual([+ { provider: PROVIDER, phase: "manual", outcome: "success", attemptsExhausted: true }+ ]);+ });++ test("records a failure", async () => {+ const { service, provider } = createService();+ provider.revoke.mockRejectedValueOnce(new Error("connection refused"));++ await service.revokeLease({ ...actor, leaseId: "lease-1" });++ expect(outcomeCalls()).toEqual([+ { provider: PROVIDER, phase: "manual", outcome: "failure", attemptsExhausted: true }+ ]);+ });++ test("records nothing when a forced revoke swallows the error", async () => {+ const { service, provider } = createService();+ provider.revoke.mockRejectedValueOnce(new Error("connection refused"));++ await service.revokeLease({ ...actor, leaseId: "lease-1", isForced: true });++ expect(outcomeCalls()).toEqual([]);+ });++ test("records a forced revoke that succeeded upstream", async () => {+ const { service } = createService();++ await service.revokeLease({ ...actor, leaseId: "lease-1", isForced: true });++ expect(outcomeCalls()).toEqual([+ { provider: PROVIDER, phase: "manual", outcome: "success", attemptsExhausted: true }+ ]);+ });+ });++ describe("lease creation cleanup", () => {+ test("counts a credential left upstream when its lease and cleanup both fail", async () => {+ const { service, provider, dynamicSecretLeaseDAL } = createService();+ dynamicSecretLeaseDAL.create.mockRejectedValueOnce(new Error("insert failed"));+ provider.revoke.mockRejectedValueOnce(new Error("connection refused"));++ await expect(service.create({ ...actor, name: "db" })).rejects.toThrow("insert failed");++ expect(vi.mocked(recordDynamicSecretUnrecordedCredentialMetric)).toHaveBeenCalledWith({ provider: PROVIDER });+ });++ test("does not count a credential the cleanup revoked", async () => {+ const { service, dynamicSecretLeaseDAL } = createService();+ dynamicSecretLeaseDAL.create.mockRejectedValueOnce(new Error("insert failed"));++ await expect(service.create({ ...actor, name: "db" })).rejects.toThrow("insert failed");++ expect(vi.mocked(recordDynamicSecretUnrecordedCredentialMetric)).not.toHaveBeenCalled();+ });+ });+});diff --git a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-service.ts b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-service.tsindex 4571f1fb7b..6c1096ba0f 100644--- a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-service.ts+++ b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-service.ts@@ -14,6 +14,10 @@ import { logger } from "@app/lib/logger";import { ms } from "@app/lib/ms";import { requestMemoKeys } from "@app/lib/request-context/memo-keys";import { requestMemoize } from "@app/lib/request-context/request-memoizer";+import {+ recordDynamicSecretRevocationOutcomeMetric,+ recordDynamicSecretUnrecordedCredentialMetric+} from "@app/lib/telemetry/metrics";import { ActorType } from "@app/services/auth/auth-type";import { TIdentityDALFactory } from "@app/services/identity/identity-dal";import { TKmsServiceFactory } from "@app/services/kms/kms-service";@@ -222,12 +226,13 @@ export const dynamicSecretLeaseServiceFactory = ({// the credential already exists upstream; without a lease row nothing would ever revoke itawait selectedProvider.revoke(decryptedStoredInput, entityId, { projectId, leaseData }, config)- .catch((revokeError: Error) =>+ .catch((revokeError: Error) => {+ recordDynamicSecretUnrecordedCredentialMetric({ provider: dynamicSecretCfg.type });logger.error(revokeError,`Failed to revoke credential after lease insert failed [dynamicSecretId=${dynamicSecretCfg.id}] [provider=${dynamicSecretCfg.type}]`- )- );+ );+ });throw error;}@@ -409,6 +414,7 @@ export const dynamicSecretLeaseServiceFactory = ({secretManagerDecryptor({ cipherTextBlob: Buffer.from(dynamicSecretCfg.encryptedInput) }).toString()) as object;+ let isRevokeErrorSwallowed = false;const revokeResponse = await selectedProvider.revoke(decryptedStoredInput,@@ -419,8 +425,19 @@ export const dynamicSecretLeaseServiceFactory = ({.catch(async (err) => {// only propogate this error if forced is falseif (!isForced) return { error: err as Error };+ isRevokeErrorSwallowed = true;});+ // A forced revoke that failed upstream is the user discarding the lease, not a revocation result.+ if (!isRevokeErrorSwallowed) {+ recordDynamicSecretRevocationOutcomeMetric({+ provider: dynamicSecretCfg.type,+ phase: "manual",+ outcome: (revokeResponse as { error?: Error })?.error ? "failure" : "success",+ attemptsExhausted: true+ });+ }+if ((revokeResponse as { error?: Error })?.error) {const { error } = revokeResponse as { error?: Error };logger.error(error?.message, "Failed to revoke lease");diff --git a/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.test.ts b/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.test.tsnew file mode 100644index 0000000000..417be0fb30--- /dev/null+++ b/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.test.ts@@ -0,0 +1,174 @@+import { beforeEach, describe, expect, test, vi } from "vitest";++import { recordSecretRotationOutcomeMetric, recordSecretRotationStatusChangeMetric } from "@app/lib/telemetry/metrics";++import { SecretRotation, SecretRotationStatus } from "./secret-rotation-v2-enums";+import { secretRotationV2ServiceFactory } from "./secret-rotation-v2-service";+import { TSecretRotationV2Raw } from "./secret-rotation-v2-types";++const rotateCredentials = vi.fn<() => Promise<unknown>>();++vi.mock("@app/lib/config/env", async (importOriginal) => ({+ ...(await importOriginal<typeof import("@app/lib/config/env")>()),+ getConfig: () => ({ isSmtpConfigured: false })+}));++vi.mock("@app/lib/logger", () => ({+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }+}));++vi.mock("@app/lib/telemetry/metrics", async (importOriginal) => ({+ ...(await importOriginal<typeof import("@app/lib/telemetry/metrics")>()),+ recordSecretRotationOutcomeMetric: vi.fn(),+ recordSecretRotationStatusChangeMetric: vi.fn()+}));++vi.mock("@app/services/app-connection/app-connection-fns", async (importOriginal) => ({+ ...(await importOriginal<typeof import("@app/services/app-connection/app-connection-fns")>()),+ decryptAppConnection: vi.fn(async () => ({}))+}));++vi.mock("@app/ee/services/secret-rotation-v2/secret-rotation-v2-fns", async (importOriginal) => ({+ ...(await importOriginal<typeof import("@app/ee/services/secret-rotation-v2/secret-rotation-v2-fns")>()),+ decryptSecretRotationCredentials: vi.fn(async () => [{}, {}])+}));++vi.mock("./stripe-api-key/stripe-api-key-rotation-fns", () => ({+ stripeApiKeyRotationFactory: () => ({ rotateCredentials })+}));++const createService = ({ lockFails = false } = {}) => {+ const release = vi.fn(async () => undefined);+ const deps = {+ keyStore: {+ acquireLock: vi.fn(async () => {+ if (lockFails) throw new Error("locked");+ return { release };+ })+ },+ kmsService: {+ createCipherPairWithDataKey: vi.fn(async () => ({+ encryptor: () => ({ cipherTextBlob: Buffer.from("encrypted") })+ }))+ },+ secretRotationV2DAL: { updateById: vi.fn(async () => ({})) },+ auditLogService: { createAuditLog: vi.fn(async () => undefined) },+ secretV2BridgeDAL: { invalidateSecretCacheByProjectId: vi.fn(async () => undefined) },+ secretQueueService: { syncSecrets: vi.fn(async () => undefined) },+ telemetryService: { sendPostHogEvents: vi.fn(async () => undefined) },+ queueService: { queue: vi.fn(async () => undefined) }+ };++ return secretRotationV2ServiceFactory(deps as never);+};++const rotation = (rotationStatus: SecretRotationStatus) =>+ ({+ id: "rotation-1",+ name: "rotation",+ type: SecretRotation.StripeApiKey,+ rotationStatus,+ projectId: "project-1",+ folderId: "folder-1",+ activeIndex: 0,+ encryptedGeneratedCredentials: Buffer.from(""),+ rotateAtUtc: { hours: 0, minutes: 0 },+ parameters: {},+ secretsMapping: {},+ connection: { id: "connection-1", orgId: "org-1", app: "stripe" },+ folder: { id: "folder-1", path: "/" },+ environment: { id: "env-1", slug: "dev", name: "Development" }+ }) as unknown as TSecretRotationV2Raw;++const outcomeCalls = () => vi.mocked(recordSecretRotationOutcomeMetric).mock.calls.map(([params]) => params);+const statusChangeCalls = () => vi.mocked(recordSecretRotationStatusChangeMetric).mock.calls.map(([params]) => params);++describe("rotateGeneratedCredentials metrics", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ });++ test("reports a manual rotation as exhausted, since it is never retried", async () => {+ rotateCredentials.mockResolvedValueOnce({});+ const service = createService();++ await service.rotateGeneratedCredentials(rotation(SecretRotationStatus.Success), { isManualRotation: true });++ expect(outcomeCalls()).toEqual([+ { type: SecretRotation.StripeApiKey, outcome: "success", attemptsExhausted: true }+ ]);+ });++ test("reports an intermediate failed attempt as not exhausted and does not flip the status", async () => {+ rotateCredentials.mockRejectedValueOnce(new Error("provider down"));+ const service = createService();++ await expect(+ service.rotateGeneratedCredentials(rotation(SecretRotationStatus.Success), { isFinalAttempt: false })+ ).rejects.toThrow();++ expect(outcomeCalls()).toEqual([+ { type: SecretRotation.StripeApiKey, outcome: "failure", attemptsExhausted: false }+ ]);+ expect(statusChangeCalls()).toEqual([]);+ });++ test("flips to failed when the final attempt fails after a previous success", async () => {+ rotateCredentials.mockRejectedValueOnce(new Error("provider down"));+ const service = createService();++ await expect(+ service.rotateGeneratedCredentials(rotation(SecretRotationStatus.Success), { isFinalAttempt: true })+ ).rejects.toThrow();++ expect(outcomeCalls()).toEqual([+ { type: SecretRotation.StripeApiKey, outcome: "failure", attemptsExhausted: true }+ ]);+ expect(statusChangeCalls()).toEqual([{ type: SecretRotation.StripeApiKey, to: "failed" }]);+ });++ test("does not flip again when an already failed rotation fails", async () => {+ rotateCredentials.mockRejectedValueOnce(new Error("provider down"));+ const service = createService();++ await expect(+ service.rotateGeneratedCredentials(rotation(SecretRotationStatus.Failed), { isFinalAttempt: true })+ ).rejects.toThrow();++ expect(statusChangeCalls()).toEqual([]);+ });++ test("flips to healthy when a previously failed rotation succeeds", async () => {+ rotateCredentials.mockResolvedValueOnce({});+ const service = createService();++ await service.rotateGeneratedCredentials(rotation(SecretRotationStatus.Failed), { isFinalAttempt: false });++ expect(outcomeCalls()).toEqual([+ { type: SecretRotation.StripeApiKey, outcome: "success", attemptsExhausted: false }+ ]);+ expect(statusChangeCalls()).toEqual([{ type: SecretRotation.StripeApiKey, to: "healthy" }]);+ });++ test("does not flip to healthy when a healthy rotation succeeds", async () => {+ rotateCredentials.mockResolvedValueOnce({});+ const service = createService();++ await service.rotateGeneratedCredentials(rotation(SecretRotationStatus.Success));++ expect(statusChangeCalls()).toEqual([]);+ });++ test("does not count lock contention as the rotation breaking", async () => {+ const service = createService({ lockFails: true });++ await expect(+ service.rotateGeneratedCredentials(rotation(SecretRotationStatus.Success), { isFinalAttempt: true })+ ).rejects.toThrow();++ expect(outcomeCalls()).toEqual([+ { type: SecretRotation.StripeApiKey, outcome: "failure", attemptsExhausted: true }+ ]);+ expect(statusChangeCalls()).toEqual([]);+ });+});diff --git a/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts b/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.tsindex 7487fc05ee..d89d898746 100644--- a/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts+++ b/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts@@ -70,7 +70,7 @@ import { getConfig } from "@app/lib/config/env";import { DatabaseErrorCode } from "@app/lib/error-codes";import { BadRequestError, DatabaseError, InternalServerError, NotFoundError } from "@app/lib/errors";import { takeRowScanWindow } from "@app/lib/fn";-import { recordSecretRotationOutcomeMetric } from "@app/lib/telemetry/metrics";+import { recordSecretRotationOutcomeMetric, recordSecretRotationStatusChangeMetric } from "@app/lib/telemetry/metrics";import { OrderByDirection, OrgServiceActor } from "@app/lib/types";import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";import { TAppConnectionDALFactory } from "@app/services/app-connection/app-connection-dal";@@ -1314,6 +1314,7 @@ export const secretRotationV2ServiceFactory = ({} = secretRotation;let lock: Awaited<ReturnType<typeof keyStore.acquireLock>> | undefined;+ const previousRotationStatus = secretRotation.rotationStatus as SecretRotationStatus;try {try {@@ -1476,13 +1477,16 @@ export const secretRotationV2ServiceFactory = ({excludeReplication: true});- recordSecretRotationOutcomeMetric({ type, outcome: "success" });+ recordSecretRotationOutcomeMetric({ type, outcome: "success", attemptsExhausted: isFinalAttempt });+ if (previousRotationStatus === SecretRotationStatus.Failed) {+ recordSecretRotationStatusChangeMetric({ type, to: "healthy" });+ }return updatedRotation;} catch (error) {const errorMessage = parseRotationErrorMessage(error);- recordSecretRotationOutcomeMetric({ type, outcome: "failure" });+ recordSecretRotationOutcomeMetric({ type, outcome: "failure", attemptsExhausted: isFinalAttempt });if (isManualRotation) {await triggerFailedWebhook(projectId, environment, error, folder, secretRotation, isManualRotation);@@ -1506,6 +1510,11 @@ export const secretRotationV2ServiceFactory = ({nextRotationAt: getNextUtcRotationInterval(secretRotation.rotateAtUtc as TSecretRotationV2["rotateAtUtc"])});+ // Without the lock another run of this rotation was in progress, which says nothing about the provider.+ if (previousRotationStatus === SecretRotationStatus.Success && lock) {+ recordSecretRotationStatusChangeMetric({ type, to: "failed" });+ }+if (shouldSendNotification) {await $queueSendSecretRotationStatusNotification(updatedRotation);await triggerFailedWebhook(projectId, environment, error, folder, secretRotation, isManualRotation);diff --git a/backend/src/keystore/keystore.ts b/backend/src/keystore/keystore.tsindex a963ba652e..00048aae60 100644--- a/backend/src/keystore/keystore.ts+++ b/backend/src/keystore/keystore.ts@@ -195,6 +195,9 @@ export const KeyStorePrefixes = {// Braces are a Redis Cluster hash tag: the index zset and every member's payload key must land on// one slot for the multi-key upsert/delete scripts.+ DynamicSecretRevocationFailing: (dynamicSecretId: string) =>+ `dynamic-secret-revocation-failing:${dynamicSecretId}` as const,+WorkerHeartbeatIndex: (workerType: string) => `worker-heartbeat:{${workerType}}` as const,WorkerHeartbeat: (workerType: string, instanceId: string) =>`worker-heartbeat:{${workerType}}:${instanceId}` as const,@@ -354,6 +357,11 @@ export type TKeyStoreFactory = {value: string | number | Buffer,prefix?: string) => Promise<"OK" | null>;+ setItemWithExpiryReturningPrevious: (+ key: string,+ expiryInSeconds: number,+ value: string | number+ ) => Promise<string | null>;deleteItem: (key: string) => Promise<number>;deleteItemsByKeyIn: (keys: string[]) => Promise<number>;deleteItems: (arg: TDeleteItems) => Promise<number>;@@ -492,6 +500,9 @@ export const keyStoreFactory = (prefix?: string) => primaryRedis.set(prefix ? `${prefix}:${key}` : key, value, "EX", expiryInSeconds, "NX");+ const setItemWithExpiryReturningPrevious = async (key: string, expiryInSeconds: number, value: string | number) =>+ primaryRedis.set(key, value, "EX", expiryInSeconds, "GET");+const deleteItem = async (key: string) => primaryRedis.del(key);const deleteItemsByKeyIn = async (keys: string[]) => {@@ -937,6 +948,7 @@ export const keyStoreFactory = (ttl,setItemWithExpiry,setItemWithExpiryNX,+ setItemWithExpiryReturningPrevious,deleteItem,deleteItems,incrementBy,diff --git a/backend/src/keystore/memory.ts b/backend/src/keystore/memory.tsindex ea55f8d22a..9709a75e60 100644--- a/backend/src/keystore/memory.ts+++ b/backend/src/keystore/memory.ts@@ -32,6 +32,11 @@ export const inMemoryKeyStore = (): TKeyStoreFactory => {store[key] = value;return "OK";},+ setItemWithExpiryReturningPrevious: async (key, _expiryInSeconds, value) => {+ const previous = store[key];+ store[key] = value;+ return previous === undefined ? null : previous.toString();+ },deleteItem: async (key) => {delete store[key];delete hashStore[key];diff --git a/backend/src/lib/telemetry/metrics.ts b/backend/src/lib/telemetry/metrics.tsindex 551a78b59c..b5f3b5d24f 100644--- a/backend/src/lib/telemetry/metrics.ts+++ b/backend/src/lib/telemetry/metrics.ts@@ -705,7 +705,7 @@ export const recordSecretWriteMetric = (params: { operation: "create" | "update"// -- Secret sync outcome (InfisicalCore meter) ----------------------------------------------------export const secretSyncOutcomeCounter = infisicalCoreMeter.createCounter("infisical.secret_sync.outcome.count", {description:- "Secret sync attempts by destination, operation, and outcome. Alert on failure ratio > 50% over 15m with >= 10 attempts, grouped by destination.",+ "Secret sync attempts by destination, operation, and outcome. A failure with attempts.exhausted=false is an intermediate retry; a finished sync is a success or a failure with attempts.exhausted=true.",unit: "{attempt}"});@@ -724,21 +724,60 @@ export const recordSecretSyncOutcomeMetric = (params: {});};+export const secretSyncStatusChangeCounter = infisicalCoreMeter.createCounter(+ "infisical.secret_sync.status_change.count",+ {+ description:+ "Secret syncs whose finished result flipped between healthy and failed, by destination. Counts distinct configs breaking without a per-config label.",+ unit: "{change}"+ }+);++export const recordSecretSyncStatusChangeMetric = (params: { destination: string; to: "failed" | "healthy" }) => {+ if (!isTelemetryEnabled()) return;+ secretSyncStatusChangeCounter.add(1, {+ destination: params.destination,+ to: params.to+ });+};+// -- Secret rotation outcome (InfisicalCore meter) --------------------------------------------------export const secretRotationOutcomeCounter = infisicalCoreMeter.createCounter("infisical.secret_rotation.outcome.count",{description:- "Secret rotation attempts by type and outcome. Alert on failure ratio > 50% per type with >= 5 total rotations.",+ "Secret rotation attempts by type and outcome. A failure with attempts.exhausted=false is an intermediate retry; a finished rotation is a success or a failure with attempts.exhausted=true.",unit: "{attempt}"});-export const recordSecretRotationOutcomeMetric = (params: { type: string; outcome: "success" | "failure" }) => {+export const recordSecretRotationOutcomeMetric = (params: {+ type: string;+ outcome: "success" | "failure";+ attemptsExhausted: boolean;+}) => {if (!isTelemetryEnabled()) return;secretRotationOutcomeCounter.add(1, {type: params.type,- outcome: params.outcome+ outcome: params.outcome,+ "attempts.exhausted": String(params.attemptsExhausted)+ });+};++export const secretRotationStatusChangeCounter = infisicalCoreMeter.createCounter(+ "infisical.secret_rotation.status_change.count",+ {+ description:+ "Secret rotations whose finished result flipped between healthy and failed, by type. Counts distinct configs breaking without a per-config label.",+ unit: "{change}"+ }+);++export const recordSecretRotationStatusChangeMetric = (params: { type: string; to: "failed" | "healthy" }) => {+ if (!isTelemetryEnabled()) return;+ secretRotationStatusChangeCounter.add(1, {+ type: params.type,+ to: params.to});};@@ -746,14 +785,72 @@ export const recordSecretRotationOutcomeMetric = (params: { type: string; outcomexport const dynamicSecretOrphanedLeaseCounter = infisicalCoreMeter.createCounter("infisical.dynamic_secret.orphaned_lease.count",{- description: "Dynamic secret lease revocation failures by provider. Alert on any value > 0 sustained 60m.",+ description:+ "Dynamic secret leases left unrevoked after every automatic revocation attempt failed, by provider. Each is a live credential that needs manual cleanup.",unit: "{failure}"});-export const recordDynamicSecretOrphanedLeaseMetric = (params: { provider: string }) => {+export const recordDynamicSecretOrphanedLeaseMetric = (params: { provider: string; count?: number }) => {+ if (!isTelemetryEnabled()) return;+ dynamicSecretOrphanedLeaseCounter.add(params.count ?? 1, {+ provider: params.provider+ });+};++export const dynamicSecretRevocationOutcomeCounter = infisicalCoreMeter.createCounter(+ "infisical.dynamic_secret.revocation.outcome.count",+ {+ description:+ "Dynamic secret lease revocations by provider, phase, and outcome. The scheduled phase records exactly one outcome per expired lease; retry records every attempt, with attempts.exhausted=true on the last.",+ unit: "{revocation}"+ }+);++export const recordDynamicSecretRevocationOutcomeMetric = (params: {+ provider: string;+ phase: "scheduled" | "retry" | "manual" | "pruning";+ outcome: "success" | "failure";+ attemptsExhausted: boolean;+}) => {+ if (!isTelemetryEnabled()) return;+ dynamicSecretRevocationOutcomeCounter.add(1, {+ provider: params.provider,+ phase: params.phase,+ outcome: params.outcome,+ "attempts.exhausted": String(params.attemptsExhausted)+ });+};++export const dynamicSecretStatusChangeCounter = infisicalCoreMeter.createCounter(+ "infisical.dynamic_secret.status_change.count",+ {+ description:+ "Dynamic secret configs whose scheduled lease revocations flipped between healthy and failed, by provider. Counts distinct configs breaking without a per-config label.",+ unit: "{change}"+ }+);++export const recordDynamicSecretStatusChangeMetric = (params: { provider: string; to: "failed" | "healthy" }) => {+ if (!isTelemetryEnabled()) return;+ dynamicSecretStatusChangeCounter.add(1, {+ provider: params.provider,+ to: params.to+ });+};++export const dynamicSecretUnrecordedCredentialCounter = infisicalCoreMeter.createCounter(+ "infisical.dynamic_secret.unrecorded_credential.count",+ {+ description:+ "Dynamic secret credentials created upstream whose lease failed to persist and whose cleanup revocation also failed, by provider. Each is a live credential with no lease to revoke it.",+ unit: "{credential}"+ }+);++export const recordDynamicSecretUnrecordedCredentialMetric = (params: { provider: string }) => {if (!isTelemetryEnabled()) return;- dynamicSecretOrphanedLeaseCounter.add(1, {+ dynamicSecretUnrecordedCredentialCounter.add(1, {provider: params.provider});};diff --git a/backend/src/lib/telemetry/telemetry-attributes.ts b/backend/src/lib/telemetry/telemetry-attributes.tsindex d522c0e476..cbb70efc6e 100644--- a/backend/src/lib/telemetry/telemetry-attributes.ts+++ b/backend/src/lib/telemetry/telemetry-attributes.ts@@ -49,6 +49,9 @@ export const INFISICAL_CORE_METER_ATTRIBUTES = ["destination","type","operation",+ // Closed enums on the secrets status_change and revocation outcome counters.+ "to",+ "phase",// Build info gauge labels — single-value per deploy, no cardinality concern"service.version","git.commit.sha",diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.tsindex 1582f3cf54..00855401db 100644--- a/backend/src/server/routes/index.ts+++ b/backend/src/server/routes/index.ts@@ -3106,7 +3106,8 @@ export const registerRoutes = async (userDAL,identityDAL,projectMembershipDAL,- projectDAL+ projectDAL,+ keyStore});const dynamicSecretService = dynamicSecretServiceFactory({orgDAL,diff --git a/backend/src/services/secret-sync/secret-sync-queue.test.ts b/backend/src/services/secret-sync/secret-sync-queue.test.tsnew file mode 100644index 0000000000..0c29bd15af--- /dev/null+++ b/backend/src/services/secret-sync/secret-sync-queue.test.ts@@ -0,0 +1,221 @@+import { beforeEach, describe, expect, test, vi } from "vitest";++import { recordSecretSyncOutcomeMetric, recordSecretSyncStatusChangeMetric } from "@app/lib/telemetry/metrics";+import { QueueJobs, QueueName } from "@app/queue";++import { SecretSync, SecretSyncInitialSyncBehavior } from "./secret-sync-enums";+import { SecretSyncError } from "./secret-sync-errors";+import { SecretSyncFns } from "./secret-sync-fns";+import { secretSyncQueueFactory } from "./secret-sync-queue";++vi.mock("@app/lib/config/env", async (importOriginal) => ({+ ...(await importOriginal<typeof import("@app/lib/config/env")>()),+ getConfig: () => ({ OTEL_TELEMETRY_COLLECTION_ENABLED: false, isSmtpConfigured: false })+}));++vi.mock("@app/lib/logger", () => ({+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }+}));++vi.mock("@app/lib/telemetry/metrics", async (importOriginal) => ({+ ...(await importOriginal<typeof import("@app/lib/telemetry/metrics")>()),+ recordSecretSyncOutcomeMetric: vi.fn(),+ recordSecretSyncStatusChangeMetric: vi.fn()+}));++vi.mock("@app/services/app-connection/app-connection-fns", async (importOriginal) => ({+ ...(await importOriginal<typeof import("@app/services/app-connection/app-connection-fns")>()),+ decryptAppConnectionCredentials: vi.fn(async () => ({}))+}));++vi.mock("@app/services/secret-sync/secret-sync-recursive-fns", async (importOriginal) => ({+ ...(await importOriginal<typeof import("@app/services/secret-sync/secret-sync-recursive-fns")>()),+ buildSyncPayload: vi.fn(async () => ({}))+}));++vi.mock("@app/services/secret-sync/secret-sync-fns", async (importOriginal) => {+ const actual = await importOriginal<typeof import("@app/services/secret-sync/secret-sync-fns")>();+ return {+ ...actual,+ enterpriseSyncCheck: vi.fn(async () => undefined),+ SecretSyncFns: { ...actual.SecretSyncFns, syncSecrets: vi.fn() }+ };+});++const ATTEMPTS = 5;+// Matches REQUEUE_LIMIT in the queue: at this count a lock failure stops requeueing and fails the sync.+const REQUEUE_LIMIT = 120;+const LAST_SYNCED_AT = new Date("2026-10-01T00:00:00.000Z");++type TSyncState = { lastSyncedAt: Date | null; lastSyncMessage: string | null };+type THandler = (job: unknown) => Promise<void>;++const createHarness = (initial: TSyncState, { lockFails = false } = {}) => {+ const handlers = new Map<string, THandler>();+ const queueService = {+ start: vi.fn((name: string, fn: THandler) => {+ handlers.set(name, fn);+ }),+ queue: vi.fn(async () => undefined)+ };++ const state = {+ id: "sync-1",+ name: "sync",+ destination: SecretSync.Railway,+ projectId: "project-1",+ folderId: "folder-1",+ connectionId: "connection-1",+ connection: { orgId: "org-1", encryptedCredentials: Buffer.from(""), projectId: "project-1" },+ environment: { slug: "dev", name: "Development" },+ folder: { path: "/" },+ syncOptions: { initialSyncBehavior: SecretSyncInitialSyncBehavior.OverwriteDestination },+ destinationConfig: {},+ ...initial+ };+ const secretSyncDAL = {+ // Every attempt reloads the sync, as the real handler does.+ findById: vi.fn(async () => ({ ...state })),+ updateById: vi.fn(async (_id: string, data: Partial<typeof state>) => {+ Object.entries(data).forEach(([key, value]) => {+ if (value !== undefined) Object.assign(state, { [key]: value });+ });+ return { ...state };+ })+ };++ secretSyncQueueFactory({+ queueService,+ secretSyncDAL,+ keyStore: {+ acquireLock: vi.fn(async () => {+ if (lockFails) throw new Error("locked");+ return { release: vi.fn(async () => undefined) };+ }),+ incrementByAndRefreshExpiryIfUnderLimit: vi.fn(async () => 1),+ decrementByOrDelete: vi.fn(async () => 0)+ },+ kmsService: { createCipherPairWithDataKey: vi.fn(async () => ({ decryptor: () => Buffer.from("") })) },+ auditLogService: { createAuditLog: vi.fn(async () => undefined) },+ telemetryService: { sendPostHogEvents: vi.fn(async () => undefined) },+ licenseService: { getPlan: vi.fn() }+ } as never);++ const runSync = (attempt: number, data: Record<string, unknown> = {}) =>+ handlers.get(QueueName.AppConnectionSecretSync)!({+ name: QueueJobs.SecretSyncSyncSecrets,+ id: `job-${attempt}`,+ data: { syncId: "sync-1", ...data },+ attemptsStarted: attempt,+ opts: { attempts: ATTEMPTS }+ });++ return { state, runSync };+};++const syncSecrets = vi.mocked(SecretSyncFns.syncSecrets);+const statusChangeCalls = () => vi.mocked(recordSecretSyncStatusChangeMetric).mock.calls.map(([params]) => params);++const healthy: TSyncState = { lastSyncedAt: LAST_SYNCED_AT, lastSyncMessage: null };+const failed: TSyncState = { lastSyncedAt: LAST_SYNCED_AT, lastSyncMessage: "Railway disabled updates" };+const neverSucceeded: TSyncState = { lastSyncedAt: null, lastSyncMessage: "Invalid token" };+const brandNew: TSyncState = { lastSyncedAt: null, lastSyncMessage: null };++describe("secret sync status change", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ });++ test("flips to failed when a previously healthy sync fails its final attempt", async () => {+ syncSecrets.mockRejectedValue(new Error("upstream 500"));+ const { runSync } = createHarness(healthy);++ await expect(runSync(ATTEMPTS)).rejects.toThrow("upstream 500");++ expect(statusChangeCalls()).toEqual([{ destination: SecretSync.Railway, to: "failed" }]);+ });++ test("uses the result before the job, not the Running status, across retries", async () => {+ syncSecrets.mockRejectedValue(new Error("upstream 500"));+ const { runSync } = createHarness(healthy);++ for (let attempt = 1; attempt < ATTEMPTS; attempt += 1) {+ // eslint-disable-next-line no-await-in-loop+ await expect(runSync(attempt)).rejects.toThrow();+ }+ expect(statusChangeCalls()).toEqual([]);+ expect(vi.mocked(recordSecretSyncOutcomeMetric)).toHaveBeenCalledTimes(ATTEMPTS - 1);++ await expect(runSync(ATTEMPTS)).rejects.toThrow();+ expect(statusChangeCalls()).toEqual([{ destination: SecretSync.Railway, to: "failed" }]);+ });++ test("flips to failed when a non-retryable error ends a healthy sync early", async () => {+ syncSecrets.mockRejectedValue(new SecretSyncError({ message: "Forbidden", shouldRetry: false }));+ const { runSync } = createHarness(healthy);++ await runSync(1);++ expect(statusChangeCalls()).toEqual([{ destination: SecretSync.Railway, to: "failed" }]);+ });++ test("does not count a sync that has never succeeded", async () => {+ syncSecrets.mockRejectedValue(new Error("invalid token"));++ await expect(createHarness(neverSucceeded).runSync(ATTEMPTS)).rejects.toThrow();+ await expect(createHarness(brandNew).runSync(ATTEMPTS)).rejects.toThrow();++ expect(statusChangeCalls()).toEqual([]);+ });++ test("does not flip again when an already failing sync fails", async () => {+ syncSecrets.mockRejectedValue(new Error("upstream 500"));+ const { runSync } = createHarness(failed);++ await expect(runSync(ATTEMPTS)).rejects.toThrow();++ expect(statusChangeCalls()).toEqual([]);+ });++ test("flips to healthy when a failing sync succeeds, and only once", async () => {+ syncSecrets.mockResolvedValue({} as never);+ const { runSync } = createHarness(failed);++ await runSync(1);+ await runSync(1);++ expect(statusChangeCalls()).toEqual([{ destination: SecretSync.Railway, to: "healthy" }]);+ });++ test("does not flip to healthy when a healthy sync succeeds", async () => {+ syncSecrets.mockResolvedValue({} as never);++ await createHarness(healthy).runSync(1);+ await createHarness(brandNew).runSync(1);++ expect(statusChangeCalls()).toEqual([]);+ });++ test("records no status change on lock contention, and does not let it mask a later breakage", async () => {+ const contended = createHarness(healthy, { lockFails: true });++ await contended.runSync(1, { failedToAcquireLockCount: REQUEUE_LIMIT });++ expect(vi.mocked(recordSecretSyncOutcomeMetric)).toHaveBeenCalledWith({+ destination: SecretSync.Railway,+ operation: "sync",+ outcome: "failure",+ attemptsExhausted: true+ });+ expect(statusChangeCalls()).toEqual([]);++ syncSecrets.mockRejectedValue(new Error("upstream 500"));+ const afterContention = createHarness({+ lastSyncedAt: contended.state.lastSyncedAt,+ lastSyncMessage: contended.state.lastSyncMessage+ });+ await expect(afterContention.runSync(ATTEMPTS)).rejects.toThrow();++ expect(statusChangeCalls()).toEqual([{ destination: SecretSync.Railway, to: "failed" }]);+ });+});diff --git a/backend/src/services/secret-sync/secret-sync-queue.ts b/backend/src/services/secret-sync/secret-sync-queue.tsindex a09cca8754..07959e5e7c 100644--- a/backend/src/services/secret-sync/secret-sync-queue.ts+++ b/backend/src/services/secret-sync/secret-sync-queue.ts@@ -11,7 +11,11 @@ import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";import { getConfig } from "@app/lib/config/env";import { CronJobName, TCronJobFactory } from "@app/lib/cron/cron-job";import { logger } from "@app/lib/logger";-import { highCardinalityMeter, recordSecretSyncOutcomeMetric } from "@app/lib/telemetry/metrics";+import {+ highCardinalityMeter,+ recordSecretSyncOutcomeMetric,+ recordSecretSyncStatusChangeMetric+} from "@app/lib/telemetry/metrics";import { triggerWorkflowIntegrationNotification } from "@app/lib/workflow-integrations/trigger-notification";import { TriggerFeature } from "@app/lib/workflow-integrations/types";import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";@@ -86,6 +90,20 @@ const DEFAULT_SECRET_SYNC_RETRY_CONFIG = {backoff: { type: "exponential" as const, delay: 3000 }};+const LOCK_FAILURE_MESSAGE =+ "Failed to run job. This typically happens when a sync is already in progress. Please try again.";++// syncStatus can't tell us how the previous run ended: every attempt overwrites it with Running first. lastSyncedAt+// is only written on success, and lastSyncMessage is cleared on success and set on a final failure, so together they+// hold the last finished result across retries. A lock-contention message says nothing about the destination.+const getLastSyncResult = ({+ lastSyncedAt,+ lastSyncMessage+}: Pick<TSecretSyncRaw, "lastSyncedAt" | "lastSyncMessage">): "succeeded" | "failed" | null => {+ if (lastSyncMessage && lastSyncMessage !== LOCK_FAILURE_MESSAGE) return "failed";+ return lastSyncedAt ? "succeeded" : null;+};+export type TSecretSyncQueueFactory = ReturnType<typeof secretSyncQueueFactory>;type TSecretSyncQueueFactoryDep = {@@ -644,6 +662,14 @@ export const secretSyncQueueFactory = ({lastSyncedAt: isSynced ? ranAt : undefined});+ const lastSyncResult = getLastSyncResult(secretSync);+ if (isSynced && lastSyncResult === "failed") {+ recordSecretSyncStatusChangeMetric({ destination: secretSync.destination, to: "healthy" });+ } else if (!isSynced && lastSyncResult === "succeeded") {+ // A sync that has never succeeded is a new misconfiguration, not a breakage, so it is not counted.+ recordSecretSyncStatusChangeMetric({ destination: secretSync.destination, to: "failed" });+ }+if (!isSynced) {void telemetryService.sendPostHogEvents({@@ -1109,8 +1135,7 @@ export const secretSyncQueueFactory = ({const secretSync = await secretSyncDAL.updateById(syncId, {syncStatus: SecretSyncStatus.Failed,- lastSyncMessage:- "Failed to run job. This typically happens when a sync is already in progress. Please try again.",+ lastSyncMessage: LOCK_FAILURE_MESSAGE,lastSyncJobId: job.id});@@ -1133,8 +1158,7 @@ export const secretSyncQueueFactory = ({case QueueJobs.SecretSyncImportSecrets: {const secretSync = await secretSyncDAL.updateById(syncId, {importStatus: SecretSyncStatus.Failed,- lastImportMessage:- "Failed to run job. This typically happens when a sync is already in progress. Please try again.",+ lastImportMessage: LOCK_FAILURE_MESSAGE,lastImportJobId: job.id});@@ -1156,8 +1180,7 @@ export const secretSyncQueueFactory = ({case QueueJobs.SecretSyncRemoveSecrets: {const secretSync = await secretSyncDAL.updateById(syncId, {removeStatus: SecretSyncStatus.Failed,- lastRemoveMessage:- "Failed to run job. This typically happens when a sync is already in progress. Please try again.",+ lastRemoveMessage: LOCK_FAILURE_MESSAGE,lastRemoveJobId: job.id});