Skip to content

Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.

feat(telemetry): enhance dynamic secret revocation metrics and add new key store functionality

Victor Hugo dos Santosccdd1aa5976c
diff --git a/backend/e2e-test/mocks/keystore.ts b/backend/e2e-test/mocks/keystore.ts
index e68c7e26a0..9ae56053b1 100644
--- a/backend/e2e-test/mocks/keystore.ts
+++ b/backend/e2e-test/mocks/keystore.ts
@@ -30,6 +30,11 @@ export const mockKeyStore = (): TKeyStoreFactory => {
store[key] = value;
return "OK";
},
+ setItemWithExpiryReturningPrevious: async (key, _expiryInSeconds, value) => {
+ const previous = store[key];
+ store[key] = value;
+ return previous === undefined ? null : previous.toString();
+ },
deleteItem: async (key) => {
delete store[key];
delete hashStore[key];
diff --git a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-queue.test.ts b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-queue.test.ts
new file mode 100644
index 0000000000..7c4665f664
--- /dev/null
+++ b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-queue.test.ts
@@ -0,0 +1,333 @@
+import { beforeEach, describe, expect, test, vi } from "vitest";
+
+import { KeyStorePrefixes } from "@app/keystore/keystore";
+import { inMemoryKeyStore } from "@app/keystore/memory";
+import { logger } from "@app/lib/logger";
+import {
+ isTelemetryEnabled,
+ recordDynamicSecretOrphanedLeaseMetric,
+ recordDynamicSecretRevocationOutcomeMetric,
+ recordDynamicSecretStatusChangeMetric
+} from "@app/lib/telemetry/metrics";
+import { QueueJobs, QueueName } from "@app/queue";
+
+import { DynamicSecretStatus } from "../dynamic-secret/dynamic-secret-types";
+import { DynamicSecretProviders } from "../dynamic-secret/providers/models";
+import { dynamicSecretLeaseQueueServiceFactory } from "./dynamic-secret-lease-queue";
+
+vi.mock("@app/lib/config/env", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("@app/lib/config/env")>()),
+ getConfig: () => ({ isDevelopmentMode: false })
+}));
+
+vi.mock("@app/lib/logger", () => ({
+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }
+}));
+
+vi.mock("@app/lib/telemetry/metrics", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("@app/lib/telemetry/metrics")>()),
+ isTelemetryEnabled: vi.fn(() => true),
+ recordDynamicSecretOrphanedLeaseMetric: vi.fn(),
+ recordDynamicSecretRevocationOutcomeMetric: vi.fn(),
+ recordDynamicSecretStatusChangeMetric: vi.fn()
+}));
+
+const PROVIDER = DynamicSecretProviders.SqlDatabase;
+const CONFIG_ID = "dynamic-secret-1";
+const MAX_ATTEMPTS = 3;
+
+const dynamicSecretCfg = {
+ id: CONFIG_ID,
+ type: PROVIDER,
+ folderId: "folder-1",
+ encryptedInput: Buffer.from("{}"),
+ status: DynamicSecretStatus.Deleting
+};
+
+const lease = (id: string) => ({
+ id,
+ dynamicSecretId: CONFIG_ID,
+ externalEntityId: `entity-${id}`,
+ config: {},
+ encryptedLeaseData: null,
+ dynamicSecret: dynamicSecretCfg
+});
+
+type THandler = (job: unknown) => Promise<void>;
+
+const createHarness = () => {
+ const handlers = new Map<string, THandler>();
+ const queueService = {
+ start: vi.fn((name: string, fn: THandler) => {
+ handlers.set(name, fn);
+ }),
+ queue: vi.fn(async () => undefined),
+ stopJobById: vi.fn(async () => undefined)
+ };
+
+ const leases = new Map<string, ReturnType<typeof lease>>();
+ const dynamicSecretLeaseDAL = {
+ findById: vi.fn(async (id: string) => leases.get(id)),
+ deleteById: vi.fn(async (id: string) => {
+ leases.delete(id);
+ }),
+ find: vi.fn(async () => [...leases.values()]),
+ updateById: vi.fn(async () => ({}))
+ };
+ const cfg = { ...dynamicSecretCfg };
+ const dynamicSecretDAL = {
+ findById: vi.fn(async () => cfg),
+ deleteById: vi.fn(async () => cfg),
+ updateById: vi.fn(async (_id: string, data: Partial<typeof cfg>) => Object.assign(cfg, data))
+ };
+ const revoke = vi.fn<(inputs: unknown, entityId: string) => Promise<{ entityId: string }>>(
+ async (_inputs, entityId) => ({ entityId })
+ );
+ const keyStore = inMemoryKeyStore();
+
+ dynamicSecretLeaseQueueServiceFactory({
+ queueService: queueService as never,
+ dynamicSecretLeaseDAL: dynamicSecretLeaseDAL as never,
+ dynamicSecretDAL: dynamicSecretDAL as never,
+ dynamicSecretProviders: { [PROVIDER]: { revoke } } as never,
+ folderDAL: { findById: vi.fn(async () => ({ id: "folder-1", projectId: "project-1" })) } as never,
+ kmsService: {
+ createCipherPairWithDataKey: vi.fn(async () => ({ decryptor: () => Buffer.from("{}") }))
+ } as never,
+ keyStore,
+ smtpService: {} as never,
+ userDAL: {} as never,
+ identityDAL: {} as never,
+ projectMembershipDAL: {} as never,
+ projectDAL: {} as never
+ });
+
+ const run = (job: { name: string; id: string; data: unknown; attempt?: number; attempts?: number }) =>
+ handlers.get(QueueName.DynamicSecretRevocation)!({
+ name: job.name,
+ id: job.id,
+ data: job.data,
+ attemptsMade: (job.attempt ?? 1) - 1,
+ opts: { attempts: job.attempts ?? MAX_ATTEMPTS }
+ });
+
+ const runScheduled = (leaseId: string, attempt = 1) =>
+ run({
+ name: QueueJobs.DynamicSecretRevocation,
+ id: leaseId,
+ data: { leaseId, dynamicSecretId: CONFIG_ID },
+ attempt
+ });
+
+ const runRetry = (leaseId: string, attempt = 1) =>
+ run({
+ name: QueueJobs.DynamicSecretRevocation,
+ id: `${leaseId}-retry`,
+ data: { leaseId, dynamicSecretId: CONFIG_ID, isRetry: true },
+ attempt
+ });
+
+ const runPruning = (attempt = 1) =>
+ run({
+ name: QueueJobs.DynamicSecretPruning,
+ id: `dynamic-secret-lease-pruning-${CONFIG_ID}`,
+ data: { dynamicSecretCfgId: CONFIG_ID },
+ attempt
+ });
+
+ const addLease = (id: string) => leases.set(id, lease(id));
+
+ return { queueService, revoke, keyStore, addLease, runScheduled, runRetry, runPruning };
+};
+
+const outcomeCalls = () => vi.mocked(recordDynamicSecretRevocationOutcomeMetric).mock.calls.map(([params]) => params);
+const statusChangeCalls = () => vi.mocked(recordDynamicSecretStatusChangeMetric).mock.calls.map(([params]) => params);
+const orphanedCalls = () => vi.mocked(recordDynamicSecretOrphanedLeaseMetric).mock.calls.map(([params]) => params);
+const retryQueued = (queue: ReturnType<typeof createHarness>["queueService"]["queue"]) =>
+ queue.mock.calls.filter((call) => (call as unknown[])[1] === QueueJobs.DynamicSecretRevocation);
+
+const failingKey = KeyStorePrefixes.DynamicSecretRevocationFailing(CONFIG_ID);
+
+describe("dynamic secret lease revocation metrics", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ vi.mocked(isTelemetryEnabled).mockReturnValue(true);
+ });
+
+ describe("scheduled phase", () => {
+ test("records one success and no status change for a healthy config", async () => {
+ const harness = createHarness();
+ harness.addLease("lease-1");
+
+ await harness.runScheduled("lease-1");
+
+ expect(outcomeCalls()).toEqual([
+ { provider: PROVIDER, phase: "scheduled", outcome: "success", attemptsExhausted: false }
+ ]);
+ expect(statusChangeCalls()).toEqual([]);
+ });
+
+ test("records exactly one failure per lease, on the final scheduled attempt", async () => {
+ const harness = createHarness();
+ harness.addLease("lease-1");
+ harness.revoke.mockRejectedValue(new Error("connection refused"));
+
+ await expect(harness.runScheduled("lease-1", 1)).rejects.toThrow("connection refused");
+ await expect(harness.runScheduled("lease-1", 2)).rejects.toThrow("connection refused");
+ expect(outcomeCalls()).toEqual([]);
+ // Handing off before the last attempt would let the retry job race the scheduled retries.
+ expect(retryQueued(harness.queueService.queue)).toHaveLength(0);
+
+ await expect(harness.runScheduled("lease-1", 3)).rejects.toThrow("connection refused");
+ expect(outcomeCalls()).toEqual([
+ { provider: PROVIDER, phase: "scheduled", outcome: "failure", attemptsExhausted: true }
+ ]);
+ expect(retryQueued(harness.queueService.queue)).toHaveLength(1);
+ });
+
+ test("records the success of a later scheduled attempt as the lease's only outcome", async () => {
+ const harness = createHarness();
+ harness.addLease("lease-1");
+ harness.revoke.mockRejectedValueOnce(new Error("timeout"));
+
+ await expect(harness.runScheduled("lease-1", 1)).rejects.toThrow("timeout");
+ await harness.runScheduled("lease-1", 2);
+
+ expect(outcomeCalls()).toEqual([
+ { provider: PROVIDER, phase: "scheduled", outcome: "success", attemptsExhausted: false }
+ ]);
+ });
+
+ test("flips to failed on a config's first failed lease and not on the next", async () => {
+ const harness = createHarness();
+ harness.addLease("lease-1");
+ harness.addLease("lease-2");
+ harness.revoke.mockRejectedValue(new Error("connection refused"));
+
+ await expect(harness.runScheduled("lease-1", MAX_ATTEMPTS)).rejects.toThrow();
+ await expect(harness.runScheduled("lease-2", MAX_ATTEMPTS)).rejects.toThrow();
+
+ expect(statusChangeCalls()).toEqual([{ provider: PROVIDER, to: "failed" }]);
+ expect(await harness.keyStore.getItem(failingKey)).toBe("1");
+ });
+
+ test("flips to healthy once when a failing config revokes again", async () => {
+ const harness = createHarness();
+ harness.addLease("lease-1");
+ harness.addLease("lease-2");
+ harness.addLease("lease-3");
+ harness.revoke.mockRejectedValueOnce(new Error("connection refused"));
+
+ await expect(harness.runScheduled("lease-1", MAX_ATTEMPTS)).rejects.toThrow();
+ await harness.runScheduled("lease-2");
+ await harness.runScheduled("lease-3");
+
+ expect(statusChangeCalls()).toEqual([
+ { provider: PROVIDER, to: "failed" },
+ { provider: PROVIDER, to: "healthy" }
+ ]);
+ expect(await harness.keyStore.getItem(failingKey)).toBeNull();
+ });
+
+ test("does not let a Redis error fail or change the revocation", async () => {
+ const harness = createHarness();
+ harness.addLease("lease-1");
+ harness.addLease("lease-2");
+ const redisError = new Error("redis unavailable");
+ vi.spyOn(harness.keyStore, "getItem").mockRejectedValue(redisError);
+ vi.spyOn(harness.keyStore, "setItemWithExpiryReturningPrevious").mockRejectedValue(redisError);
+
+ await expect(harness.runScheduled("lease-1")).resolves.toBeUndefined();
+
+ harness.revoke.mockRejectedValueOnce(new Error("connection refused"));
+ await expect(harness.runScheduled("lease-2", MAX_ATTEMPTS)).rejects.toThrow("connection refused");
+
+ expect(statusChangeCalls()).toEqual([]);
+ expect(outcomeCalls().map(({ outcome }) => outcome)).toEqual(["success", "failure"]);
+ expect(retryQueued(harness.queueService.queue)).toHaveLength(1);
+ expect(vi.mocked(logger.warn)).toHaveBeenCalledTimes(2);
+ });
+
+ test("skips the status change entirely while telemetry is disabled", async () => {
+ vi.mocked(isTelemetryEnabled).mockReturnValue(false);
+ const harness = createHarness();
+ harness.addLease("lease-1");
+ const setPrevious = vi.spyOn(harness.keyStore, "setItemWithExpiryReturningPrevious");
+ harness.revoke.mockRejectedValue(new Error("connection refused"));
+
+ await expect(harness.runScheduled("lease-1", MAX_ATTEMPTS)).rejects.toThrow();
+
+ expect(setPrevious).not.toHaveBeenCalled();
+ });
+
+ test("records nothing when the lease no longer exists", async () => {
+ const harness = createHarness();
+
+ await harness.runScheduled("missing-lease");
+
+ expect(outcomeCalls()).toEqual([]);
+ expect(statusChangeCalls()).toEqual([]);
+ });
+ });
+
+ describe("retry phase", () => {
+ test("records every attempt, exhausted only on the last, and orphans the lease there", async () => {
+ const harness = createHarness();
+ harness.addLease("lease-1");
+ harness.revoke.mockRejectedValue(new Error("connection refused"));
+
+ for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt += 1) {
+ // eslint-disable-next-line no-await-in-loop
+ await expect(harness.runRetry("lease-1", attempt)).rejects.toThrow();
+ }
+
+ expect(outcomeCalls()).toEqual([
+ { provider: PROVIDER, phase: "retry", outcome: "failure", attemptsExhausted: false },
+ { provider: PROVIDER, phase: "retry", outcome: "failure", attemptsExhausted: false },
+ { provider: PROVIDER, phase: "retry", outcome: "failure", attemptsExhausted: true }
+ ]);
+ expect(orphanedCalls()).toEqual([{ provider: PROVIDER }]);
+ expect(statusChangeCalls()).toEqual([]);
+ });
+ });
+
+ describe("pruning", () => {
+ test("records each lease once and orphans only the ones left unrevoked", async () => {
+ const harness = createHarness();
+ harness.addLease("lease-1");
+ harness.addLease("lease-2");
+ harness.addLease("lease-3");
+ harness.revoke.mockImplementation(async (_inputs, entityId) => {
+ if (entityId === "entity-lease-2") throw new Error("permission denied");
+ return { entityId };
+ });
+
+ await expect(harness.runPruning(1)).rejects.toThrow("permission denied");
+ // The config is now FailedDeletion, so BullMQ's remaining attempts return without revoking.
+ await harness.runPruning(2);
+ await harness.runPruning(3);
+
+ expect(outcomeCalls()).toEqual([
+ { provider: PROVIDER, phase: "pruning", outcome: "success", attemptsExhausted: true },
+ { provider: PROVIDER, phase: "pruning", outcome: "failure", attemptsExhausted: true },
+ { provider: PROVIDER, phase: "pruning", outcome: "success", attemptsExhausted: true }
+ ]);
+ expect(orphanedCalls()).toEqual([{ provider: PROVIDER, count: 1 }]);
+ expect(harness.revoke).toHaveBeenCalledTimes(3);
+ });
+
+ test("records a success per lease when every revoke succeeds", async () => {
+ const harness = createHarness();
+ harness.addLease("lease-1");
+ harness.addLease("lease-2");
+
+ await harness.runPruning(1);
+
+ expect(outcomeCalls()).toEqual([
+ { provider: PROVIDER, phase: "pruning", outcome: "success", attemptsExhausted: false },
+ { provider: PROVIDER, phase: "pruning", outcome: "success", attemptsExhausted: false }
+ ]);
+ expect(orphanedCalls()).toEqual([]);
+ });
+ });
+});
diff --git a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-queue.ts b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-queue.ts
index b86519d27c..b653314668 100644
--- a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-queue.ts
+++ b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-queue.ts
@@ -1,9 +1,15 @@
import { ProjectMembershipRole } from "@app/db/schemas";
+import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";
import { getConfig } from "@app/lib/config/env";
import { applyJitter } from "@app/lib/delay";
import { DisableRotationErrors, NotFoundError } from "@app/lib/errors";
import { logger } from "@app/lib/logger";
-import { recordDynamicSecretOrphanedLeaseMetric } from "@app/lib/telemetry/metrics";
+import {
+ isTelemetryEnabled,
+ recordDynamicSecretOrphanedLeaseMetric,
+ recordDynamicSecretRevocationOutcomeMetric,
+ recordDynamicSecretStatusChangeMetric
+} from "@app/lib/telemetry/metrics";
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
import { TIdentityDALFactory } from "@app/services/identity/identity-dal";
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
@@ -33,6 +39,7 @@ type TDynamicSecretLeaseQueueServiceFactoryDep = {
dynamicSecretProviders: Record<DynamicSecretProviders, TDynamicProviderFns>;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
folderDAL: Pick<TSecretFolderDALFactory, "findById" | "findSecretPathByFolderIds">;
+ keyStore: Pick<TKeyStoreFactory, "getItem" | "deleteItem" | "setItemWithExpiryReturningPrevious">;
};
export type TDynamicSecretLeaseQueueServiceFactory = {
@@ -43,6 +50,7 @@ export type TDynamicSecretLeaseQueueServiceFactory = {
};
const MAX_REVOCATION_RETRY_COUNT = 3;
+const REVOCATION_FAILING_STATUS_EXPIRY_SECONDS = 7 * 24 * 60 * 60;
export const dynamicSecretLeaseQueueServiceFactory = ({
queueService,
@@ -53,7 +61,8 @@ export const dynamicSecretLeaseQueueServiceFactory = ({
folderDAL,
projectMembershipDAL,
projectDAL,
- smtpService
+ smtpService,
+ keyStore
}: TDynamicSecretLeaseQueueServiceFactoryDep): TDynamicSecretLeaseQueueServiceFactory => {
const pruneDynamicSecret = async (dynamicSecretCfgId: string) => {
await queueService.queue(
@@ -137,16 +146,69 @@ export const dynamicSecretLeaseQueueServiceFactory = ({
);
};
+ // Dynamic secret configs have no revocation status column, so whether a config's scheduled revocations are
+ // failing is kept in Redis. This only feeds a metric and must never fail or retry the revocation job.
+ const $recordScheduledRevocationStatusChange = async ({
+ dynamicSecretId,
+ provider,
+ isRevoked
+ }: {
+ dynamicSecretId: string;
+ provider: string;
+ isRevoked: boolean;
+ }) => {
+ if (!isTelemetryEnabled()) return;
+
+ const key = KeyStorePrefixes.DynamicSecretRevocationFailing(dynamicSecretId);
+ try {
+ if (isRevoked) {
+ if (!(await keyStore.getItem(key))) return;
+ // Only the caller that actually deleted the key reports the recovery.
+ const deleted = await keyStore.deleteItem(key);
+ if (deleted > 0) recordDynamicSecretStatusChangeMetric({ provider, to: "healthy" });
+ return;
+ }
+
+ const previous = await keyStore.setItemWithExpiryReturningPrevious(
+ key,
+ REVOCATION_FAILING_STATUS_EXPIRY_SECONDS,
+ "1"
+ );
+ if (!previous) recordDynamicSecretStatusChangeMetric({ provider, to: "failed" });
+ } catch (error) {
+ logger.warn(
+ error,
+ `Failed to record dynamic secret revocation status change [dynamicSecretId=${dynamicSecretId}] [provider=${provider}]`
+ );
+ }
+ };
+
+ const $recordPruningOutcomes = (provider: string, revoked: boolean[], attemptsExhausted: boolean) => {
+ revoked.forEach((isRevoked) =>
+ recordDynamicSecretRevocationOutcomeMetric({
+ provider,
+ phase: "pruning",
+ outcome: isRevoked ? "success" : "failure",
+ attemptsExhausted
+ })
+ );
+ };
+
const $dynamicSecretQueueJob = async (
jobName: string,
jobId: string,
data: { leaseId: string; dynamicSecretId: string; isRetry?: boolean } | { dynamicSecretCfgId: string },
- retryCount?: number
+ retryCount?: number,
+ maxAttempts = 1
): Promise<void> => {
let providerType: string | undefined;
+ let configId: string | undefined;
+ const isFinalAttempt = retryCount !== undefined && retryCount >= maxAttempts;
+ // Per-lease revoke results of this pruning run, so a failed run reports each lease once.
+ let pruningRevoked: boolean[] | undefined;
try {
if (jobName === QueueJobs.DynamicSecretRevocation) {
- const { leaseId } = data as { leaseId: string };
+ const { leaseId, isRetry } = data as { leaseId: string; isRetry?: boolean };
logger.info("Dynamic secret lease revocation started: ", leaseId, jobId);
const dynamicSecretLease = await dynamicSecretLeaseDAL.findById(leaseId);
@@ -154,9 +216,13 @@ export const dynamicSecretLeaseQueueServiceFactory = ({
return;
}
- const folder = await folderDAL.findById(dynamicSecretLease.dynamicSecret.folderId);
+ const dynamicSecretCfg = dynamicSecretLease.dynamicSecret;
+ providerType = dynamicSecretCfg.type;
+ configId = dynamicSecretCfg.id;
+
+ const folder = await folderDAL.findById(dynamicSecretCfg.folderId);
if (!folder) {
- logger.info(`Failed to find folder with ${dynamicSecretLease.dynamicSecret.folderId}`);
+ logger.info(`Failed to find folder with ${dynamicSecretCfg.folderId}`);
return;
}
@@ -165,8 +231,6 @@ export const dynamicSecretLeaseQueueServiceFactory = ({
projectId: folder.projectId
});
- const dynamicSecretCfg = dynamicSecretLease.dynamicSecret;
- providerType = dynamicSecretCfg.type;
const selectedProvider = dynamicSecretProviders[dynamicSecretCfg.type as DynamicSecretProviders];
const decryptedStoredInput = JSON.parse(
secretManagerDecryptor({ cipherTextBlob: dynamicSecretCfg.encryptedInput }).toString()
@@ -182,6 +246,20 @@ export const dynamicSecretLeaseQueueServiceFactory = ({
dynamicSecretLease.config as TDynamicSecretLeaseConfig
);
await dynamicSecretLeaseDAL.deleteById(dynamicSecretLease.id);
+
+ recordDynamicSecretRevocationOutcomeMetric({
+ provider: providerType,
+ phase: isRetry ? "retry" : "scheduled",
+ outcome: "success",
+ attemptsExhausted: isFinalAttempt
+ });
+ if (!isRetry) {
+ await $recordScheduledRevocationStatusChange({
+ dynamicSecretId: configId,
+ provider: providerType,
+ isRevoked: true
+ });
+ }
return;
}
@@ -218,8 +296,8 @@ export const dynamicSecretLeaseQueueServiceFactory = ({
) as object;
await Promise.all(dynamicSecretLeases.map(({ id }) => unsetLeaseRevocation(id)));
- await Promise.all(
- dynamicSecretLeases.map(({ externalEntityId, config, encryptedLeaseData }) =>
+ const revocations = await Promise.allSettled(
+ dynamicSecretLeases.map(async ({ externalEntityId, config, encryptedLeaseData }) =>
selectedProvider.revoke(
decryptedStoredInput,
externalEntityId,
@@ -231,9 +309,17 @@ export const dynamicSecretLeaseQueueServiceFactory = ({
)
)
);
+ pruningRevoked = revocations.map((revocation) => revocation.status === "fulfilled");
+
+ const failedRevocation = revocations.find(
+ (revocation): revocation is PromiseRejectedResult => revocation.status === "rejected"
+ );
+ if (failedRevocation) throw failedRevocation.reason;
}
await dynamicSecretDAL.deleteById(dynamicSecretCfgId);
+
+ if (pruningRevoked) $recordPruningOutcomes(providerType, pruningRevoked, isFinalAttempt);
}
logger.info("Finished dynamic secret job", jobId);
} catch (error) {
@@ -245,8 +331,13 @@ export const dynamicSecretLeaseQueueServiceFactory = ({
status: DynamicSecretStatus.FailedDeletion,
statusDetails: (error as Error)?.message?.slice(0, 255)
});
- if (providerType) {
- recordDynamicSecretOrphanedLeaseMetric({ provider: providerType });
+ if (providerType && pruningRevoked && !(error instanceof DisableRotationErrors)) {
+ $recordPruningOutcomes(providerType, pruningRevoked, true);
+
+ const unrevokedLeaseCount = pruningRevoked.filter((isRevoked) => !isRevoked).length;
+ if (unrevokedLeaseCount > 0) {
+ recordDynamicSecretOrphanedLeaseMetric({ provider: providerType, count: unrevokedLeaseCount });
+ }
}
}
@@ -256,20 +347,42 @@ export const dynamicSecretLeaseQueueServiceFactory = ({
isRetry?: boolean;
dynamicSecretId: string;
};
+ const isRevocationFailure = !(error instanceof DisableRotationErrors);
+
+ // A scheduled revocation reports one outcome per lease: the final attempt's, since BullMQ retries it.
+ if (providerType && isRevocationFailure && (isRetry || isFinalAttempt)) {
+ recordDynamicSecretRevocationOutcomeMetric({
+ provider: providerType,
+ phase: isRetry ? "retry" : "scheduled",
+ outcome: "failure",
+ attemptsExhausted: isFinalAttempt
+ });
+ if (!isRetry && configId) {
+ await $recordScheduledRevocationStatusChange({
+ dynamicSecretId: configId,
+ provider: providerType,
+ isRevoked: false
+ });
+ }
+ }
+
await dynamicSecretLeaseDAL.updateById(leaseId, {
status: DynamicSecretStatus.FailedDeletion,
statusDetails: `${(error as Error)?.message?.slice(0, 255)} - Retrying automatically`
});
// only add to retry queue if this is not a retry, and if the error is not a DisableRotationErrors error
- if (!isRetry && !(error instanceof DisableRotationErrors)) {
- // if revocation fails, we should stop the job and queue a new job to retry the revocation at a later time.
- await queueService.stopJobById(QueueName.DynamicSecretRevocation, jobId);
- await queueFailedRevocation(leaseId, dynamicSecretId);
+ if (!isRetry && isRevocationFailure) {
+ // BullMQ retries the scheduled job (it cannot be removed while active), so hand off to the slower retry
+ // queue only once its attempts are used up. Handing off earlier lets the retry job revoke the lease between
+ // scheduled attempts, leaving the lease with no scheduled outcome.
+ if (isFinalAttempt) {
+ await queueFailedRevocation(leaseId, dynamicSecretId);
+ }
// if its the last attempt, and the error isn't a DisableRotationErrors error, send an email to the project admins (debounced)
- } else if (isRetry && !(error instanceof DisableRotationErrors)) {
- if (retryCount && retryCount === MAX_REVOCATION_RETRY_COUNT) {
+ } else if (isRetry && isRevocationFailure) {
+ if (isFinalAttempt) {
// if all retries fail, we should also stop the automatic revocation job.
// the ID of the revocation job is set to the leaseId, so we can use that to stop the job
@@ -349,7 +462,7 @@ export const dynamicSecretLeaseQueueServiceFactory = ({
};
queueService.start(QueueName.DynamicSecretRevocation, async (job) => {
- await $dynamicSecretQueueJob(job.name, job.id as string, job.data, job.attemptsMade + 1);
+ await $dynamicSecretQueueJob(job.name, job.id as string, job.data, job.attemptsMade + 1, job.opts.attempts);
});
queueService.start(QueueName.DynamicSecretLeaseRevocationFailedEmail, async (job) => {
diff --git a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-service.test.ts b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-service.test.ts
new file mode 100644
index 0000000000..9058abfb98
--- /dev/null
+++ b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-service.test.ts
@@ -0,0 +1,171 @@
+import { createMongoAbility } from "@casl/ability";
+import { beforeEach, describe, expect, test, vi } from "vitest";
+
+import {
+ recordDynamicSecretRevocationOutcomeMetric,
+ recordDynamicSecretUnrecordedCredentialMetric
+} from "@app/lib/telemetry/metrics";
+import { ActorType, AuthMethod } from "@app/services/auth/auth-type";
+
+import { DynamicSecretProviders } from "../dynamic-secret/providers/models";
+import { dynamicSecretLeaseServiceFactory } from "./dynamic-secret-lease-service";
+
+vi.mock("@app/lib/config/env", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("@app/lib/config/env")>()),
+ getConfig: () => ({ MAX_LEASE_LIMIT: 100 })
+}));
+
+vi.mock("@app/lib/logger", () => ({
+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }
+}));
+
+vi.mock("@app/lib/telemetry/metrics", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("@app/lib/telemetry/metrics")>()),
+ recordDynamicSecretRevocationOutcomeMetric: vi.fn(),
+ recordDynamicSecretUnrecordedCredentialMetric: vi.fn()
+}));
+
+const PROVIDER = DynamicSecretProviders.SqlDatabase;
+
+const dynamicSecretCfg = {
+ id: "dynamic-secret-1",
+ type: PROVIDER,
+ folderId: "folder-1",
+ encryptedInput: Buffer.from("{}"),
+ defaultTTL: "1h",
+ maxTTL: null,
+ usernameTemplate: null,
+ metadata: []
+};
+
+const createService = () => {
+ const provider = {
+ create: vi.fn(async () => ({ entityId: "entity-1", data: {} })),
+ revoke: vi.fn(async () => ({ entityId: "entity-1" }))
+ };
+ const dynamicSecretLeaseDAL = {
+ countLeasesForDynamicSecret: vi.fn(async () => 0),
+ create: vi.fn(async () => ({ id: "lease-1" })),
+ findById: vi.fn(async () => ({
+ id: "lease-1",
+ dynamicSecretId: dynamicSecretCfg.id,
+ externalEntityId: "entity-1",
+ config: {},
+ encryptedLeaseData: null,
+ dynamicSecret: dynamicSecretCfg
+ })),
+ updateById: vi.fn(async () => ({})),
+ deleteById: vi.fn(async () => ({}))
+ };
+
+ const service = dynamicSecretLeaseServiceFactory({
+ dynamicSecretLeaseDAL: dynamicSecretLeaseDAL as never,
+ dynamicSecretDAL: { findOne: vi.fn(async () => dynamicSecretCfg) } as never,
+ dynamicSecretProviders: { [PROVIDER]: provider } as never,
+ dynamicSecretQueueService: {
+ setLeaseRevocation: vi.fn(async () => undefined),
+ unsetLeaseRevocation: vi.fn(async () => undefined),
+ queueFailedRevocation: vi.fn(async () => undefined)
+ } as never,
+ licenseService: { getPlan: vi.fn(async () => ({ dynamicSecret: true })) } as never,
+ folderDAL: { findBySecretPath: vi.fn(async () => ({ id: "folder-1", envId: "env-1" })) } as never,
+ permissionService: {
+ getProjectPermission: vi.fn(async () => ({
+ permission: createMongoAbility([{ action: "manage", subject: "all" }])
+ }))
+ } as never,
+ projectDAL: { findProjectBySlug: vi.fn(async () => ({ id: "project-1" })) } as never,
+ kmsService: {
+ createCipherPairWithDataKey: vi.fn(async () => ({
+ encryptor: () => ({ cipherTextBlob: Buffer.from("") }),
+ decryptor: () => Buffer.from("{}")
+ }))
+ } as never,
+ userDAL: { findById: vi.fn(async () => undefined) } as never,
+ identityDAL: { findById: vi.fn(async () => undefined) } as never,
+ secretValidationRuleService: { findConstraintsForGeneratedSecret: vi.fn(async () => ({ ruleNames: [] })) } as never
+ });
+
+ return { service, provider, dynamicSecretLeaseDAL };
+};
+
+const actor = {
+ actor: ActorType.USER,
+ actorId: "user-1",
+ actorOrgId: "org-1",
+ actorAuthMethod: AuthMethod.EMAIL,
+ projectSlug: "project",
+ environmentSlug: "dev",
+ path: "/"
+};
+
+const outcomeCalls = () => vi.mocked(recordDynamicSecretRevocationOutcomeMetric).mock.calls.map(([params]) => params);
+
+describe("dynamic secret lease service metrics", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ });
+
+ describe("manual revocation", () => {
+ test("records a success", async () => {
+ const { service } = createService();
+
+ await service.revokeLease({ ...actor, leaseId: "lease-1" });
+
+ expect(outcomeCalls()).toEqual([
+ { provider: PROVIDER, phase: "manual", outcome: "success", attemptsExhausted: true }
+ ]);
+ });
+
+ test("records a failure", async () => {
+ const { service, provider } = createService();
+ provider.revoke.mockRejectedValueOnce(new Error("connection refused"));
+
+ await service.revokeLease({ ...actor, leaseId: "lease-1" });
+
+ expect(outcomeCalls()).toEqual([
+ { provider: PROVIDER, phase: "manual", outcome: "failure", attemptsExhausted: true }
+ ]);
+ });
+
+ test("records nothing when a forced revoke swallows the error", async () => {
+ const { service, provider } = createService();
+ provider.revoke.mockRejectedValueOnce(new Error("connection refused"));
+
+ await service.revokeLease({ ...actor, leaseId: "lease-1", isForced: true });
+
+ expect(outcomeCalls()).toEqual([]);
+ });
+
+ test("records a forced revoke that succeeded upstream", async () => {
+ const { service } = createService();
+
+ await service.revokeLease({ ...actor, leaseId: "lease-1", isForced: true });
+
+ expect(outcomeCalls()).toEqual([
+ { provider: PROVIDER, phase: "manual", outcome: "success", attemptsExhausted: true }
+ ]);
+ });
+ });
+
+ describe("lease creation cleanup", () => {
+ test("counts a credential left upstream when its lease and cleanup both fail", async () => {
+ const { service, provider, dynamicSecretLeaseDAL } = createService();
+ dynamicSecretLeaseDAL.create.mockRejectedValueOnce(new Error("insert failed"));
+ provider.revoke.mockRejectedValueOnce(new Error("connection refused"));
+
+ await expect(service.create({ ...actor, name: "db" })).rejects.toThrow("insert failed");
+
+ expect(vi.mocked(recordDynamicSecretUnrecordedCredentialMetric)).toHaveBeenCalledWith({ provider: PROVIDER });
+ });
+
+ test("does not count a credential the cleanup revoked", async () => {
+ const { service, dynamicSecretLeaseDAL } = createService();
+ dynamicSecretLeaseDAL.create.mockRejectedValueOnce(new Error("insert failed"));
+
+ await expect(service.create({ ...actor, name: "db" })).rejects.toThrow("insert failed");
+
+ expect(vi.mocked(recordDynamicSecretUnrecordedCredentialMetric)).not.toHaveBeenCalled();
+ });
+ });
+});
diff --git a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-service.ts b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-service.ts
index 4571f1fb7b..6c1096ba0f 100644
--- a/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-service.ts
+++ b/backend/src/ee/services/dynamic-secret-lease/dynamic-secret-lease-service.ts
@@ -14,6 +14,10 @@ import { logger } from "@app/lib/logger";
import { ms } from "@app/lib/ms";
import { requestMemoKeys } from "@app/lib/request-context/memo-keys";
import { requestMemoize } from "@app/lib/request-context/request-memoizer";
+import {
+ recordDynamicSecretRevocationOutcomeMetric,
+ recordDynamicSecretUnrecordedCredentialMetric
+} from "@app/lib/telemetry/metrics";
import { ActorType } from "@app/services/auth/auth-type";
import { TIdentityDALFactory } from "@app/services/identity/identity-dal";
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
@@ -222,12 +226,13 @@ export const dynamicSecretLeaseServiceFactory = ({
// the credential already exists upstream; without a lease row nothing would ever revoke it
await selectedProvider
.revoke(decryptedStoredInput, entityId, { projectId, leaseData }, config)
- .catch((revokeError: Error) =>
+ .catch((revokeError: Error) => {
+ recordDynamicSecretUnrecordedCredentialMetric({ provider: dynamicSecretCfg.type });
logger.error(
revokeError,
`Failed to revoke credential after lease insert failed [dynamicSecretId=${dynamicSecretCfg.id}] [provider=${dynamicSecretCfg.type}]`
- )
- );
+ );
+ });
throw error;
}
@@ -409,6 +414,7 @@ export const dynamicSecretLeaseServiceFactory = ({
secretManagerDecryptor({ cipherTextBlob: Buffer.from(dynamicSecretCfg.encryptedInput) }).toString()
) as object;
+ let isRevokeErrorSwallowed = false;
const revokeResponse = await selectedProvider
.revoke(
decryptedStoredInput,
@@ -419,8 +425,19 @@ export const dynamicSecretLeaseServiceFactory = ({
.catch(async (err) => {
// only propogate this error if forced is false
if (!isForced) return { error: err as Error };
+ isRevokeErrorSwallowed = true;
});
+ // A forced revoke that failed upstream is the user discarding the lease, not a revocation result.
+ if (!isRevokeErrorSwallowed) {
+ recordDynamicSecretRevocationOutcomeMetric({
+ provider: dynamicSecretCfg.type,
+ phase: "manual",
+ outcome: (revokeResponse as { error?: Error })?.error ? "failure" : "success",
+ attemptsExhausted: true
+ });
+ }
+
if ((revokeResponse as { error?: Error })?.error) {
const { error } = revokeResponse as { error?: Error };
logger.error(error?.message, "Failed to revoke lease");
diff --git a/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.test.ts b/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.test.ts
new file mode 100644
index 0000000000..417be0fb30
--- /dev/null
+++ b/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.test.ts
@@ -0,0 +1,174 @@
+import { beforeEach, describe, expect, test, vi } from "vitest";
+
+import { recordSecretRotationOutcomeMetric, recordSecretRotationStatusChangeMetric } from "@app/lib/telemetry/metrics";
+
+import { SecretRotation, SecretRotationStatus } from "./secret-rotation-v2-enums";
+import { secretRotationV2ServiceFactory } from "./secret-rotation-v2-service";
+import { TSecretRotationV2Raw } from "./secret-rotation-v2-types";
+
+const rotateCredentials = vi.fn<() => Promise<unknown>>();
+
+vi.mock("@app/lib/config/env", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("@app/lib/config/env")>()),
+ getConfig: () => ({ isSmtpConfigured: false })
+}));
+
+vi.mock("@app/lib/logger", () => ({
+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }
+}));
+
+vi.mock("@app/lib/telemetry/metrics", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("@app/lib/telemetry/metrics")>()),
+ recordSecretRotationOutcomeMetric: vi.fn(),
+ recordSecretRotationStatusChangeMetric: vi.fn()
+}));
+
+vi.mock("@app/services/app-connection/app-connection-fns", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("@app/services/app-connection/app-connection-fns")>()),
+ decryptAppConnection: vi.fn(async () => ({}))
+}));
+
+vi.mock("@app/ee/services/secret-rotation-v2/secret-rotation-v2-fns", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("@app/ee/services/secret-rotation-v2/secret-rotation-v2-fns")>()),
+ decryptSecretRotationCredentials: vi.fn(async () => [{}, {}])
+}));
+
+vi.mock("./stripe-api-key/stripe-api-key-rotation-fns", () => ({
+ stripeApiKeyRotationFactory: () => ({ rotateCredentials })
+}));
+
+const createService = ({ lockFails = false } = {}) => {
+ const release = vi.fn(async () => undefined);
+ const deps = {
+ keyStore: {
+ acquireLock: vi.fn(async () => {
+ if (lockFails) throw new Error("locked");
+ return { release };
+ })
+ },
+ kmsService: {
+ createCipherPairWithDataKey: vi.fn(async () => ({
+ encryptor: () => ({ cipherTextBlob: Buffer.from("encrypted") })
+ }))
+ },
+ secretRotationV2DAL: { updateById: vi.fn(async () => ({})) },
+ auditLogService: { createAuditLog: vi.fn(async () => undefined) },
+ secretV2BridgeDAL: { invalidateSecretCacheByProjectId: vi.fn(async () => undefined) },
+ secretQueueService: { syncSecrets: vi.fn(async () => undefined) },
+ telemetryService: { sendPostHogEvents: vi.fn(async () => undefined) },
+ queueService: { queue: vi.fn(async () => undefined) }
+ };
+
+ return secretRotationV2ServiceFactory(deps as never);
+};
+
+const rotation = (rotationStatus: SecretRotationStatus) =>
+ ({
+ id: "rotation-1",
+ name: "rotation",
+ type: SecretRotation.StripeApiKey,
+ rotationStatus,
+ projectId: "project-1",
+ folderId: "folder-1",
+ activeIndex: 0,
+ encryptedGeneratedCredentials: Buffer.from(""),
+ rotateAtUtc: { hours: 0, minutes: 0 },
+ parameters: {},
+ secretsMapping: {},
+ connection: { id: "connection-1", orgId: "org-1", app: "stripe" },
+ folder: { id: "folder-1", path: "/" },
+ environment: { id: "env-1", slug: "dev", name: "Development" }
+ }) as unknown as TSecretRotationV2Raw;
+
+const outcomeCalls = () => vi.mocked(recordSecretRotationOutcomeMetric).mock.calls.map(([params]) => params);
+const statusChangeCalls = () => vi.mocked(recordSecretRotationStatusChangeMetric).mock.calls.map(([params]) => params);
+
+describe("rotateGeneratedCredentials metrics", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ });
+
+ test("reports a manual rotation as exhausted, since it is never retried", async () => {
+ rotateCredentials.mockResolvedValueOnce({});
+ const service = createService();
+
+ await service.rotateGeneratedCredentials(rotation(SecretRotationStatus.Success), { isManualRotation: true });
+
+ expect(outcomeCalls()).toEqual([
+ { type: SecretRotation.StripeApiKey, outcome: "success", attemptsExhausted: true }
+ ]);
+ });
+
+ test("reports an intermediate failed attempt as not exhausted and does not flip the status", async () => {
+ rotateCredentials.mockRejectedValueOnce(new Error("provider down"));
+ const service = createService();
+
+ await expect(
+ service.rotateGeneratedCredentials(rotation(SecretRotationStatus.Success), { isFinalAttempt: false })
+ ).rejects.toThrow();
+
+ expect(outcomeCalls()).toEqual([
+ { type: SecretRotation.StripeApiKey, outcome: "failure", attemptsExhausted: false }
+ ]);
+ expect(statusChangeCalls()).toEqual([]);
+ });
+
+ test("flips to failed when the final attempt fails after a previous success", async () => {
+ rotateCredentials.mockRejectedValueOnce(new Error("provider down"));
+ const service = createService();
+
+ await expect(
+ service.rotateGeneratedCredentials(rotation(SecretRotationStatus.Success), { isFinalAttempt: true })
+ ).rejects.toThrow();
+
+ expect(outcomeCalls()).toEqual([
+ { type: SecretRotation.StripeApiKey, outcome: "failure", attemptsExhausted: true }
+ ]);
+ expect(statusChangeCalls()).toEqual([{ type: SecretRotation.StripeApiKey, to: "failed" }]);
+ });
+
+ test("does not flip again when an already failed rotation fails", async () => {
+ rotateCredentials.mockRejectedValueOnce(new Error("provider down"));
+ const service = createService();
+
+ await expect(
+ service.rotateGeneratedCredentials(rotation(SecretRotationStatus.Failed), { isFinalAttempt: true })
+ ).rejects.toThrow();
+
+ expect(statusChangeCalls()).toEqual([]);
+ });
+
+ test("flips to healthy when a previously failed rotation succeeds", async () => {
+ rotateCredentials.mockResolvedValueOnce({});
+ const service = createService();
+
+ await service.rotateGeneratedCredentials(rotation(SecretRotationStatus.Failed), { isFinalAttempt: false });
+
+ expect(outcomeCalls()).toEqual([
+ { type: SecretRotation.StripeApiKey, outcome: "success", attemptsExhausted: false }
+ ]);
+ expect(statusChangeCalls()).toEqual([{ type: SecretRotation.StripeApiKey, to: "healthy" }]);
+ });
+
+ test("does not flip to healthy when a healthy rotation succeeds", async () => {
+ rotateCredentials.mockResolvedValueOnce({});
+ const service = createService();
+
+ await service.rotateGeneratedCredentials(rotation(SecretRotationStatus.Success));
+
+ expect(statusChangeCalls()).toEqual([]);
+ });
+
+ test("does not count lock contention as the rotation breaking", async () => {
+ const service = createService({ lockFails: true });
+
+ await expect(
+ service.rotateGeneratedCredentials(rotation(SecretRotationStatus.Success), { isFinalAttempt: true })
+ ).rejects.toThrow();
+
+ expect(outcomeCalls()).toEqual([
+ { type: SecretRotation.StripeApiKey, outcome: "failure", attemptsExhausted: true }
+ ]);
+ expect(statusChangeCalls()).toEqual([]);
+ });
+});
diff --git a/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts b/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts
index 7487fc05ee..d89d898746 100644
--- a/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts
+++ b/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts
@@ -70,7 +70,7 @@ import { getConfig } from "@app/lib/config/env";
import { DatabaseErrorCode } from "@app/lib/error-codes";
import { BadRequestError, DatabaseError, InternalServerError, NotFoundError } from "@app/lib/errors";
import { takeRowScanWindow } from "@app/lib/fn";
-import { recordSecretRotationOutcomeMetric } from "@app/lib/telemetry/metrics";
+import { recordSecretRotationOutcomeMetric, recordSecretRotationStatusChangeMetric } from "@app/lib/telemetry/metrics";
import { OrderByDirection, OrgServiceActor } from "@app/lib/types";
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
import { TAppConnectionDALFactory } from "@app/services/app-connection/app-connection-dal";
@@ -1314,6 +1314,7 @@ export const secretRotationV2ServiceFactory = ({
} = secretRotation;
let lock: Awaited<ReturnType<typeof keyStore.acquireLock>> | undefined;
+ const previousRotationStatus = secretRotation.rotationStatus as SecretRotationStatus;
try {
try {
@@ -1476,13 +1477,16 @@ export const secretRotationV2ServiceFactory = ({
excludeReplication: true
});
- recordSecretRotationOutcomeMetric({ type, outcome: "success" });
+ recordSecretRotationOutcomeMetric({ type, outcome: "success", attemptsExhausted: isFinalAttempt });
+ if (previousRotationStatus === SecretRotationStatus.Failed) {
+ recordSecretRotationStatusChangeMetric({ type, to: "healthy" });
+ }
return updatedRotation;
} catch (error) {
const errorMessage = parseRotationErrorMessage(error);
- recordSecretRotationOutcomeMetric({ type, outcome: "failure" });
+ recordSecretRotationOutcomeMetric({ type, outcome: "failure", attemptsExhausted: isFinalAttempt });
if (isManualRotation) {
await triggerFailedWebhook(projectId, environment, error, folder, secretRotation, isManualRotation);
@@ -1506,6 +1510,11 @@ export const secretRotationV2ServiceFactory = ({
nextRotationAt: getNextUtcRotationInterval(secretRotation.rotateAtUtc as TSecretRotationV2["rotateAtUtc"])
});
+ // Without the lock another run of this rotation was in progress, which says nothing about the provider.
+ if (previousRotationStatus === SecretRotationStatus.Success && lock) {
+ recordSecretRotationStatusChangeMetric({ type, to: "failed" });
+ }
+
if (shouldSendNotification) {
await $queueSendSecretRotationStatusNotification(updatedRotation);
await triggerFailedWebhook(projectId, environment, error, folder, secretRotation, isManualRotation);
diff --git a/backend/src/keystore/keystore.ts b/backend/src/keystore/keystore.ts
index a963ba652e..00048aae60 100644
--- a/backend/src/keystore/keystore.ts
+++ b/backend/src/keystore/keystore.ts
@@ -195,6 +195,9 @@ export const KeyStorePrefixes = {
// Braces are a Redis Cluster hash tag: the index zset and every member's payload key must land on
// one slot for the multi-key upsert/delete scripts.
+ DynamicSecretRevocationFailing: (dynamicSecretId: string) =>
+ `dynamic-secret-revocation-failing:${dynamicSecretId}` as const,
+
WorkerHeartbeatIndex: (workerType: string) => `worker-heartbeat:{${workerType}}` as const,
WorkerHeartbeat: (workerType: string, instanceId: string) =>
`worker-heartbeat:{${workerType}}:${instanceId}` as const,
@@ -354,6 +357,11 @@ export type TKeyStoreFactory = {
value: string | number | Buffer,
prefix?: string
) => Promise<"OK" | null>;
+ setItemWithExpiryReturningPrevious: (
+ key: string,
+ expiryInSeconds: number,
+ value: string | number
+ ) => Promise<string | null>;
deleteItem: (key: string) => Promise<number>;
deleteItemsByKeyIn: (keys: string[]) => Promise<number>;
deleteItems: (arg: TDeleteItems) => Promise<number>;
@@ -492,6 +500,9 @@ export const keyStoreFactory = (
prefix?: string
) => primaryRedis.set(prefix ? `${prefix}:${key}` : key, value, "EX", expiryInSeconds, "NX");
+ const setItemWithExpiryReturningPrevious = async (key: string, expiryInSeconds: number, value: string | number) =>
+ primaryRedis.set(key, value, "EX", expiryInSeconds, "GET");
+
const deleteItem = async (key: string) => primaryRedis.del(key);
const deleteItemsByKeyIn = async (keys: string[]) => {
@@ -937,6 +948,7 @@ export const keyStoreFactory = (
ttl,
setItemWithExpiry,
setItemWithExpiryNX,
+ setItemWithExpiryReturningPrevious,
deleteItem,
deleteItems,
incrementBy,
diff --git a/backend/src/keystore/memory.ts b/backend/src/keystore/memory.ts
index ea55f8d22a..9709a75e60 100644
--- a/backend/src/keystore/memory.ts
+++ b/backend/src/keystore/memory.ts
@@ -32,6 +32,11 @@ export const inMemoryKeyStore = (): TKeyStoreFactory => {
store[key] = value;
return "OK";
},
+ setItemWithExpiryReturningPrevious: async (key, _expiryInSeconds, value) => {
+ const previous = store[key];
+ store[key] = value;
+ return previous === undefined ? null : previous.toString();
+ },
deleteItem: async (key) => {
delete store[key];
delete hashStore[key];
diff --git a/backend/src/lib/telemetry/metrics.ts b/backend/src/lib/telemetry/metrics.ts
index 551a78b59c..b5f3b5d24f 100644
--- a/backend/src/lib/telemetry/metrics.ts
+++ b/backend/src/lib/telemetry/metrics.ts
@@ -705,7 +705,7 @@ export const recordSecretWriteMetric = (params: { operation: "create" | "update"
// -- Secret sync outcome (InfisicalCore meter) ----------------------------------------------------
export const secretSyncOutcomeCounter = infisicalCoreMeter.createCounter("infisical.secret_sync.outcome.count", {
description:
- "Secret sync attempts by destination, operation, and outcome. Alert on failure ratio > 50% over 15m with >= 10 attempts, grouped by destination.",
+ "Secret sync attempts by destination, operation, and outcome. A failure with attempts.exhausted=false is an intermediate retry; a finished sync is a success or a failure with attempts.exhausted=true.",
unit: "{attempt}"
});
@@ -724,21 +724,60 @@ export const recordSecretSyncOutcomeMetric = (params: {
});
};
+export const secretSyncStatusChangeCounter = infisicalCoreMeter.createCounter(
+ "infisical.secret_sync.status_change.count",
+ {
+ description:
+ "Secret syncs whose finished result flipped between healthy and failed, by destination. Counts distinct configs breaking without a per-config label.",
+ unit: "{change}"
+ }
+);
+
+export const recordSecretSyncStatusChangeMetric = (params: { destination: string; to: "failed" | "healthy" }) => {
+ if (!isTelemetryEnabled()) return;
+ secretSyncStatusChangeCounter.add(1, {
+ destination: params.destination,
+ to: params.to
+ });
+};
+
// -- Secret rotation outcome (InfisicalCore meter) --------------------------------------------------
export const secretRotationOutcomeCounter = infisicalCoreMeter.createCounter(
"infisical.secret_rotation.outcome.count",
{
description:
- "Secret rotation attempts by type and outcome. Alert on failure ratio > 50% per type with >= 5 total rotations.",
+ "Secret rotation attempts by type and outcome. A failure with attempts.exhausted=false is an intermediate retry; a finished rotation is a success or a failure with attempts.exhausted=true.",
unit: "{attempt}"
}
);
-export const recordSecretRotationOutcomeMetric = (params: { type: string; outcome: "success" | "failure" }) => {
+export const recordSecretRotationOutcomeMetric = (params: {
+ type: string;
+ outcome: "success" | "failure";
+ attemptsExhausted: boolean;
+}) => {
if (!isTelemetryEnabled()) return;
secretRotationOutcomeCounter.add(1, {
type: params.type,
- outcome: params.outcome
+ outcome: params.outcome,
+ "attempts.exhausted": String(params.attemptsExhausted)
+ });
+};
+
+export const secretRotationStatusChangeCounter = infisicalCoreMeter.createCounter(
+ "infisical.secret_rotation.status_change.count",
+ {
+ description:
+ "Secret rotations whose finished result flipped between healthy and failed, by type. Counts distinct configs breaking without a per-config label.",
+ unit: "{change}"
+ }
+);
+
+export const recordSecretRotationStatusChangeMetric = (params: { type: string; to: "failed" | "healthy" }) => {
+ if (!isTelemetryEnabled()) return;
+ secretRotationStatusChangeCounter.add(1, {
+ type: params.type,
+ to: params.to
});
};
@@ -746,14 +785,72 @@ export const recordSecretRotationOutcomeMetric = (params: { type: string; outcom
export const dynamicSecretOrphanedLeaseCounter = infisicalCoreMeter.createCounter(
"infisical.dynamic_secret.orphaned_lease.count",
{
- description: "Dynamic secret lease revocation failures by provider. Alert on any value > 0 sustained 60m.",
+ description:
+ "Dynamic secret leases left unrevoked after every automatic revocation attempt failed, by provider. Each is a live credential that needs manual cleanup.",
unit: "{failure}"
}
);
-export const recordDynamicSecretOrphanedLeaseMetric = (params: { provider: string }) => {
+export const recordDynamicSecretOrphanedLeaseMetric = (params: { provider: string; count?: number }) => {
+ if (!isTelemetryEnabled()) return;
+ dynamicSecretOrphanedLeaseCounter.add(params.count ?? 1, {
+ provider: params.provider
+ });
+};
+
+export const dynamicSecretRevocationOutcomeCounter = infisicalCoreMeter.createCounter(
+ "infisical.dynamic_secret.revocation.outcome.count",
+ {
+ description:
+ "Dynamic secret lease revocations by provider, phase, and outcome. The scheduled phase records exactly one outcome per expired lease; retry records every attempt, with attempts.exhausted=true on the last.",
+ unit: "{revocation}"
+ }
+);
+
+export const recordDynamicSecretRevocationOutcomeMetric = (params: {
+ provider: string;
+ phase: "scheduled" | "retry" | "manual" | "pruning";
+ outcome: "success" | "failure";
+ attemptsExhausted: boolean;
+}) => {
+ if (!isTelemetryEnabled()) return;
+ dynamicSecretRevocationOutcomeCounter.add(1, {
+ provider: params.provider,
+ phase: params.phase,
+ outcome: params.outcome,
+ "attempts.exhausted": String(params.attemptsExhausted)
+ });
+};
+
+export const dynamicSecretStatusChangeCounter = infisicalCoreMeter.createCounter(
+ "infisical.dynamic_secret.status_change.count",
+ {
+ description:
+ "Dynamic secret configs whose scheduled lease revocations flipped between healthy and failed, by provider. Counts distinct configs breaking without a per-config label.",
+ unit: "{change}"
+ }
+);
+
+export const recordDynamicSecretStatusChangeMetric = (params: { provider: string; to: "failed" | "healthy" }) => {
+ if (!isTelemetryEnabled()) return;
+ dynamicSecretStatusChangeCounter.add(1, {
+ provider: params.provider,
+ to: params.to
+ });
+};
+
+export const dynamicSecretUnrecordedCredentialCounter = infisicalCoreMeter.createCounter(
+ "infisical.dynamic_secret.unrecorded_credential.count",
+ {
+ description:
+ "Dynamic secret credentials created upstream whose lease failed to persist and whose cleanup revocation also failed, by provider. Each is a live credential with no lease to revoke it.",
+ unit: "{credential}"
+ }
+);
+
+export const recordDynamicSecretUnrecordedCredentialMetric = (params: { provider: string }) => {
if (!isTelemetryEnabled()) return;
- dynamicSecretOrphanedLeaseCounter.add(1, {
+ dynamicSecretUnrecordedCredentialCounter.add(1, {
provider: params.provider
});
};
diff --git a/backend/src/lib/telemetry/telemetry-attributes.ts b/backend/src/lib/telemetry/telemetry-attributes.ts
index d522c0e476..cbb70efc6e 100644
--- a/backend/src/lib/telemetry/telemetry-attributes.ts
+++ b/backend/src/lib/telemetry/telemetry-attributes.ts
@@ -49,6 +49,9 @@ export const INFISICAL_CORE_METER_ATTRIBUTES = [
"destination",
"type",
"operation",
+ // Closed enums on the secrets status_change and revocation outcome counters.
+ "to",
+ "phase",
// Build info gauge labels — single-value per deploy, no cardinality concern
"service.version",
"git.commit.sha",
diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts
index 1582f3cf54..00855401db 100644
--- a/backend/src/server/routes/index.ts
+++ b/backend/src/server/routes/index.ts
@@ -3106,7 +3106,8 @@ export const registerRoutes = async (
userDAL,
identityDAL,
projectMembershipDAL,
- projectDAL
+ projectDAL,
+ keyStore
});
const dynamicSecretService = dynamicSecretServiceFactory({
orgDAL,
diff --git a/backend/src/services/secret-sync/secret-sync-queue.test.ts b/backend/src/services/secret-sync/secret-sync-queue.test.ts
new file mode 100644
index 0000000000..0c29bd15af
--- /dev/null
+++ b/backend/src/services/secret-sync/secret-sync-queue.test.ts
@@ -0,0 +1,221 @@
+import { beforeEach, describe, expect, test, vi } from "vitest";
+
+import { recordSecretSyncOutcomeMetric, recordSecretSyncStatusChangeMetric } from "@app/lib/telemetry/metrics";
+import { QueueJobs, QueueName } from "@app/queue";
+
+import { SecretSync, SecretSyncInitialSyncBehavior } from "./secret-sync-enums";
+import { SecretSyncError } from "./secret-sync-errors";
+import { SecretSyncFns } from "./secret-sync-fns";
+import { secretSyncQueueFactory } from "./secret-sync-queue";
+
+vi.mock("@app/lib/config/env", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("@app/lib/config/env")>()),
+ getConfig: () => ({ OTEL_TELEMETRY_COLLECTION_ENABLED: false, isSmtpConfigured: false })
+}));
+
+vi.mock("@app/lib/logger", () => ({
+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }
+}));
+
+vi.mock("@app/lib/telemetry/metrics", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("@app/lib/telemetry/metrics")>()),
+ recordSecretSyncOutcomeMetric: vi.fn(),
+ recordSecretSyncStatusChangeMetric: vi.fn()
+}));
+
+vi.mock("@app/services/app-connection/app-connection-fns", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("@app/services/app-connection/app-connection-fns")>()),
+ decryptAppConnectionCredentials: vi.fn(async () => ({}))
+}));
+
+vi.mock("@app/services/secret-sync/secret-sync-recursive-fns", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("@app/services/secret-sync/secret-sync-recursive-fns")>()),
+ buildSyncPayload: vi.fn(async () => ({}))
+}));
+
+vi.mock("@app/services/secret-sync/secret-sync-fns", async (importOriginal) => {
+ const actual = await importOriginal<typeof import("@app/services/secret-sync/secret-sync-fns")>();
+ return {
+ ...actual,
+ enterpriseSyncCheck: vi.fn(async () => undefined),
+ SecretSyncFns: { ...actual.SecretSyncFns, syncSecrets: vi.fn() }
+ };
+});
+
+const ATTEMPTS = 5;
+// Matches REQUEUE_LIMIT in the queue: at this count a lock failure stops requeueing and fails the sync.
+const REQUEUE_LIMIT = 120;
+const LAST_SYNCED_AT = new Date("2026-10-01T00:00:00.000Z");
+
+type TSyncState = { lastSyncedAt: Date | null; lastSyncMessage: string | null };
+type THandler = (job: unknown) => Promise<void>;
+
+const createHarness = (initial: TSyncState, { lockFails = false } = {}) => {
+ const handlers = new Map<string, THandler>();
+ const queueService = {
+ start: vi.fn((name: string, fn: THandler) => {
+ handlers.set(name, fn);
+ }),
+ queue: vi.fn(async () => undefined)
+ };
+
+ const state = {
+ id: "sync-1",
+ name: "sync",
+ destination: SecretSync.Railway,
+ projectId: "project-1",
+ folderId: "folder-1",
+ connectionId: "connection-1",
+ connection: { orgId: "org-1", encryptedCredentials: Buffer.from(""), projectId: "project-1" },
+ environment: { slug: "dev", name: "Development" },
+ folder: { path: "/" },
+ syncOptions: { initialSyncBehavior: SecretSyncInitialSyncBehavior.OverwriteDestination },
+ destinationConfig: {},
+ ...initial
+ };
+ const secretSyncDAL = {
+ // Every attempt reloads the sync, as the real handler does.
+ findById: vi.fn(async () => ({ ...state })),
+ updateById: vi.fn(async (_id: string, data: Partial<typeof state>) => {
+ Object.entries(data).forEach(([key, value]) => {
+ if (value !== undefined) Object.assign(state, { [key]: value });
+ });
+ return { ...state };
+ })
+ };
+
+ secretSyncQueueFactory({
+ queueService,
+ secretSyncDAL,
+ keyStore: {
+ acquireLock: vi.fn(async () => {
+ if (lockFails) throw new Error("locked");
+ return { release: vi.fn(async () => undefined) };
+ }),
+ incrementByAndRefreshExpiryIfUnderLimit: vi.fn(async () => 1),
+ decrementByOrDelete: vi.fn(async () => 0)
+ },
+ kmsService: { createCipherPairWithDataKey: vi.fn(async () => ({ decryptor: () => Buffer.from("") })) },
+ auditLogService: { createAuditLog: vi.fn(async () => undefined) },
+ telemetryService: { sendPostHogEvents: vi.fn(async () => undefined) },
+ licenseService: { getPlan: vi.fn() }
+ } as never);
+
+ const runSync = (attempt: number, data: Record<string, unknown> = {}) =>
+ handlers.get(QueueName.AppConnectionSecretSync)!({
+ name: QueueJobs.SecretSyncSyncSecrets,
+ id: `job-${attempt}`,
+ data: { syncId: "sync-1", ...data },
+ attemptsStarted: attempt,
+ opts: { attempts: ATTEMPTS }
+ });
+
+ return { state, runSync };
+};
+
+const syncSecrets = vi.mocked(SecretSyncFns.syncSecrets);
+const statusChangeCalls = () => vi.mocked(recordSecretSyncStatusChangeMetric).mock.calls.map(([params]) => params);
+
+const healthy: TSyncState = { lastSyncedAt: LAST_SYNCED_AT, lastSyncMessage: null };
+const failed: TSyncState = { lastSyncedAt: LAST_SYNCED_AT, lastSyncMessage: "Railway disabled updates" };
+const neverSucceeded: TSyncState = { lastSyncedAt: null, lastSyncMessage: "Invalid token" };
+const brandNew: TSyncState = { lastSyncedAt: null, lastSyncMessage: null };
+
+describe("secret sync status change", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ });
+
+ test("flips to failed when a previously healthy sync fails its final attempt", async () => {
+ syncSecrets.mockRejectedValue(new Error("upstream 500"));
+ const { runSync } = createHarness(healthy);
+
+ await expect(runSync(ATTEMPTS)).rejects.toThrow("upstream 500");
+
+ expect(statusChangeCalls()).toEqual([{ destination: SecretSync.Railway, to: "failed" }]);
+ });
+
+ test("uses the result before the job, not the Running status, across retries", async () => {
+ syncSecrets.mockRejectedValue(new Error("upstream 500"));
+ const { runSync } = createHarness(healthy);
+
+ for (let attempt = 1; attempt < ATTEMPTS; attempt += 1) {
+ // eslint-disable-next-line no-await-in-loop
+ await expect(runSync(attempt)).rejects.toThrow();
+ }
+ expect(statusChangeCalls()).toEqual([]);
+ expect(vi.mocked(recordSecretSyncOutcomeMetric)).toHaveBeenCalledTimes(ATTEMPTS - 1);
+
+ await expect(runSync(ATTEMPTS)).rejects.toThrow();
+ expect(statusChangeCalls()).toEqual([{ destination: SecretSync.Railway, to: "failed" }]);
+ });
+
+ test("flips to failed when a non-retryable error ends a healthy sync early", async () => {
+ syncSecrets.mockRejectedValue(new SecretSyncError({ message: "Forbidden", shouldRetry: false }));
+ const { runSync } = createHarness(healthy);
+
+ await runSync(1);
+
+ expect(statusChangeCalls()).toEqual([{ destination: SecretSync.Railway, to: "failed" }]);
+ });
+
+ test("does not count a sync that has never succeeded", async () => {
+ syncSecrets.mockRejectedValue(new Error("invalid token"));
+
+ await expect(createHarness(neverSucceeded).runSync(ATTEMPTS)).rejects.toThrow();
+ await expect(createHarness(brandNew).runSync(ATTEMPTS)).rejects.toThrow();
+
+ expect(statusChangeCalls()).toEqual([]);
+ });
+
+ test("does not flip again when an already failing sync fails", async () => {
+ syncSecrets.mockRejectedValue(new Error("upstream 500"));
+ const { runSync } = createHarness(failed);
+
+ await expect(runSync(ATTEMPTS)).rejects.toThrow();
+
+ expect(statusChangeCalls()).toEqual([]);
+ });
+
+ test("flips to healthy when a failing sync succeeds, and only once", async () => {
+ syncSecrets.mockResolvedValue({} as never);
+ const { runSync } = createHarness(failed);
+
+ await runSync(1);
+ await runSync(1);
+
+ expect(statusChangeCalls()).toEqual([{ destination: SecretSync.Railway, to: "healthy" }]);
+ });
+
+ test("does not flip to healthy when a healthy sync succeeds", async () => {
+ syncSecrets.mockResolvedValue({} as never);
+
+ await createHarness(healthy).runSync(1);
+ await createHarness(brandNew).runSync(1);
+
+ expect(statusChangeCalls()).toEqual([]);
+ });
+
+ test("records no status change on lock contention, and does not let it mask a later breakage", async () => {
+ const contended = createHarness(healthy, { lockFails: true });
+
+ await contended.runSync(1, { failedToAcquireLockCount: REQUEUE_LIMIT });
+
+ expect(vi.mocked(recordSecretSyncOutcomeMetric)).toHaveBeenCalledWith({
+ destination: SecretSync.Railway,
+ operation: "sync",
+ outcome: "failure",
+ attemptsExhausted: true
+ });
+ expect(statusChangeCalls()).toEqual([]);
+
+ syncSecrets.mockRejectedValue(new Error("upstream 500"));
+ const afterContention = createHarness({
+ lastSyncedAt: contended.state.lastSyncedAt,
+ lastSyncMessage: contended.state.lastSyncMessage
+ });
+ await expect(afterContention.runSync(ATTEMPTS)).rejects.toThrow();
+
+ expect(statusChangeCalls()).toEqual([{ destination: SecretSync.Railway, to: "failed" }]);
+ });
+});
diff --git a/backend/src/services/secret-sync/secret-sync-queue.ts b/backend/src/services/secret-sync/secret-sync-queue.ts
index a09cca8754..07959e5e7c 100644
--- a/backend/src/services/secret-sync/secret-sync-queue.ts
+++ b/backend/src/services/secret-sync/secret-sync-queue.ts
@@ -11,7 +11,11 @@ import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";
import { getConfig } from "@app/lib/config/env";
import { CronJobName, TCronJobFactory } from "@app/lib/cron/cron-job";
import { logger } from "@app/lib/logger";
-import { highCardinalityMeter, recordSecretSyncOutcomeMetric } from "@app/lib/telemetry/metrics";
+import {
+ highCardinalityMeter,
+ recordSecretSyncOutcomeMetric,
+ recordSecretSyncStatusChangeMetric
+} from "@app/lib/telemetry/metrics";
import { triggerWorkflowIntegrationNotification } from "@app/lib/workflow-integrations/trigger-notification";
import { TriggerFeature } from "@app/lib/workflow-integrations/types";
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
@@ -86,6 +90,20 @@ const DEFAULT_SECRET_SYNC_RETRY_CONFIG = {
backoff: { type: "exponential" as const, delay: 3000 }
};
+const LOCK_FAILURE_MESSAGE =
+ "Failed to run job. This typically happens when a sync is already in progress. Please try again.";
+
+// syncStatus can't tell us how the previous run ended: every attempt overwrites it with Running first. lastSyncedAt
+// is only written on success, and lastSyncMessage is cleared on success and set on a final failure, so together they
+// hold the last finished result across retries. A lock-contention message says nothing about the destination.
+const getLastSyncResult = ({
+ lastSyncedAt,
+ lastSyncMessage
+}: Pick<TSecretSyncRaw, "lastSyncedAt" | "lastSyncMessage">): "succeeded" | "failed" | null => {
+ if (lastSyncMessage && lastSyncMessage !== LOCK_FAILURE_MESSAGE) return "failed";
+ return lastSyncedAt ? "succeeded" : null;
+};
+
export type TSecretSyncQueueFactory = ReturnType<typeof secretSyncQueueFactory>;
type TSecretSyncQueueFactoryDep = {
@@ -644,6 +662,14 @@ export const secretSyncQueueFactory = ({
lastSyncedAt: isSynced ? ranAt : undefined
});
+ const lastSyncResult = getLastSyncResult(secretSync);
+ if (isSynced && lastSyncResult === "failed") {
+ recordSecretSyncStatusChangeMetric({ destination: secretSync.destination, to: "healthy" });
+ } else if (!isSynced && lastSyncResult === "succeeded") {
+ // A sync that has never succeeded is a new misconfiguration, not a breakage, so it is not counted.
+ recordSecretSyncStatusChangeMetric({ destination: secretSync.destination, to: "failed" });
+ }
+
if (!isSynced) {
void telemetryService
.sendPostHogEvents({
@@ -1109,8 +1135,7 @@ export const secretSyncQueueFactory = ({
const secretSync = await secretSyncDAL.updateById(syncId, {
syncStatus: SecretSyncStatus.Failed,
- lastSyncMessage:
- "Failed to run job. This typically happens when a sync is already in progress. Please try again.",
+ lastSyncMessage: LOCK_FAILURE_MESSAGE,
lastSyncJobId: job.id
});
@@ -1133,8 +1158,7 @@ export const secretSyncQueueFactory = ({
case QueueJobs.SecretSyncImportSecrets: {
const secretSync = await secretSyncDAL.updateById(syncId, {
importStatus: SecretSyncStatus.Failed,
- lastImportMessage:
- "Failed to run job. This typically happens when a sync is already in progress. Please try again.",
+ lastImportMessage: LOCK_FAILURE_MESSAGE,
lastImportJobId: job.id
});
@@ -1156,8 +1180,7 @@ export const secretSyncQueueFactory = ({
case QueueJobs.SecretSyncRemoveSecrets: {
const secretSync = await secretSyncDAL.updateById(syncId, {
removeStatus: SecretSyncStatus.Failed,
- lastRemoveMessage:
- "Failed to run job. This typically happens when a sync is already in progress. Please try again.",
+ lastRemoveMessage: LOCK_FAILURE_MESSAGE,
lastRemoveJobId: job.id
});