Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.
TypeScript acmeagentsai-agentscertificate-managementdevsecopsenvironment-variableskubernetesopen-sourcepampkipostgresprivate-caprivileged-access-managementsecret-managementsecret-managersecretssecrets-managementsecuritysecurity-toolsvault
refactor(secret-value-tracking): run the backfill as one job
Calvinb8e7bfec5809
diff --git a/backend/e2e-test/routes/v1/org-secret-value-tracking.spec.ts b/backend/e2e-test/routes/v1/org-secret-value-tracking.spec.tsindex e4364133a9..a5dbd9f623 100644--- a/backend/e2e-test/routes/v1/org-secret-value-tracking.spec.ts+++ b/backend/e2e-test/routes/v1/org-secret-value-tracking.spec.ts@@ -93,6 +93,21 @@ const waitForCompletion = async (authToken: string, timeoutMs = 30_000) => {}};+// A backfill is refused while one is already in flight for the same scope, so a test that drives a+// run to completion cannot share an organization with the next one: the previous job may still be+// finishing when the next enable arrives, and would swallow it. Each such test gets its own.+const withOwnOrg = async (+ name: string,+ body: (ctx: Awaited<ReturnType<typeof createIsolatedOrgAndProject>>) => Promise<void>+) => {+ const ctx = await createIsolatedOrgAndProject(name);+ try {+ await body(ctx);+ } finally {+ await ctx.cleanup();+ }+};+describe("Org-wide secret value tracking", () => {// The walk is a queue round trip per chunk, so it takes longer than the 5s default.vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 });@@ -116,77 +131,77 @@ describe("Org-wide secret value tracking", () => {// Re-running a completed backfill is the repair path for every way an unindexed row can get back// into a finished org, so it has to be accepted rather than refused as "already enabled".- test("a completed organization can run the backfill again", async () => {- expect((await enable(authToken)).statusCode).toBe(200);- await waitForCompletion(authToken);- expect((await status(authToken)).status).toBe("completed");- });-- test("searching by value refuses while tracking is off, and works once the backfill completes", async () => {- await createSecretV2({- workspaceId: projectId,- environmentSlug: ENV,- secretPath: "/",- key: "PREDATES_TRACKING",- value: "written-before-tracking",- authToken- });-- await makeOrgIncomplete(orgId);-- const refused = await searchByValue("written-before-tracking", authToken);- expect(refused.statusCode).toBe(400);- expect(refused.json().message).toMatch(/org-wide secret value tracking/i);-- expect((await status(authToken)).status).toBe("not-found");-- const started = await enable(authToken);- expect(started.statusCode).toBe(200);-- const finished = await waitForCompletion(authToken);- expect(finished.projectsTotal).toBeGreaterThanOrEqual(1);- expect(finished.projectsDone).toBeGreaterThanOrEqual(1);-- const found = await searchByValue("written-before-tracking", authToken);- expect(found.statusCode).toBe(200);- expect(found.json().secrets.map((s: { key: string }) => s.key)).toEqual(["PREDATES_TRACKING"]);- });+ test("a completed organization can run the backfill again", async () =>+ withOwnOrg("org-value-tracking-rerun", async (ctx) => {+ expect((await enable(ctx.authToken)).statusCode).toBe(200);+ await waitForCompletion(ctx.authToken);+ expect((await status(ctx.authToken)).status).toBe("completed");+ }));++ test("searching by value refuses while tracking is off, and works once the backfill completes", async () =>+ withOwnOrg("org-value-tracking-search", async (ctx) => {+ await createSecretV2({+ workspaceId: ctx.projectId,+ environmentSlug: ENV,+ secretPath: "/",+ key: "PREDATES_TRACKING",+ value: "written-before-tracking",+ authToken: ctx.authToken+ });++ await makeOrgIncomplete(ctx.orgId);++ const refused = await searchByValue("written-before-tracking", ctx.authToken);+ expect(refused.statusCode).toBe(400);+ expect(refused.json().message).toMatch(/org-wide secret value tracking/i);++ expect((await enable(ctx.authToken)).statusCode).toBe(200);++ const finished = await waitForCompletion(ctx.authToken);+ expect(finished.projectsTotal).toBeGreaterThanOrEqual(1);+ expect(finished.projectsDone).toBeGreaterThanOrEqual(1);++ const found = await searchByValue("written-before-tracking", ctx.authToken);+ expect(found.statusCode).toBe(200);+ expect(found.json().secrets.map((secret: { key: string }) => secret.key)).toEqual(["PREDATES_TRACKING"]);+ }));// The (key, id) cursor exists for this: the unique index on (key, folderId) is partial on- // type = 'shared', so a personal override shares a key with the shared secret in its folder and a+ // type = 'shared', so a personal override shares a key with the shared secret beside it and a// key-only cursor would walk past one of them.- test("a personal override sharing a key with a shared secret is backfilled too", async () => {- await createSecretV2({- workspaceId: projectId,- environmentSlug: ENV,- secretPath: "/",- key: "SHARED_AND_PERSONAL",- value: "the-shared-value",- authToken- });- await createSecretV2({- workspaceId: projectId,- environmentSlug: ENV,- secretPath: "/",- key: "SHARED_AND_PERSONAL",- value: "the-personal-value",- type: SecretType.Personal,- authToken- });-- await makeOrgIncomplete(orgId);- expect((await enable(authToken)).statusCode).toBe(200);- await waitForCompletion(authToken);-- const rows = await testDb(TableName.SecretV2)- .where({ key: "SHARED_AND_PERSONAL" })- .select("id", "type", "secretValueOrgBlindIndex");-- expect(rows).toHaveLength(2);- rows.forEach((row) => expect(row.secretValueOrgBlindIndex).toEqual(expect.any(String)));- // Two different values must not collide on one digest.- expect(new Set(rows.map((row) => row.secretValueOrgBlindIndex)).size).toBe(2);- });+ test("a personal override sharing a key with a shared secret is backfilled too", async () =>+ withOwnOrg("org-value-tracking-personal", async (ctx) => {+ await createSecretV2({+ workspaceId: ctx.projectId,+ environmentSlug: ENV,+ secretPath: "/",+ key: "SHARED_AND_PERSONAL",+ value: "the-shared-value",+ authToken: ctx.authToken+ });+ await createSecretV2({+ workspaceId: ctx.projectId,+ environmentSlug: ENV,+ secretPath: "/",+ key: "SHARED_AND_PERSONAL",+ value: "the-personal-value",+ type: SecretType.Personal,+ authToken: ctx.authToken+ });++ await makeOrgIncomplete(ctx.orgId);+ expect((await enable(ctx.authToken)).statusCode).toBe(200);+ await waitForCompletion(ctx.authToken);++ const rows = await testDb(TableName.SecretV2)+ .where({ key: "SHARED_AND_PERSONAL" })+ .select("id", "type", "secretValueOrgBlindIndex");++ expect(rows).toHaveLength(2);+ rows.forEach((row) => expect(row.secretValueOrgBlindIndex).toEqual(expect.any(String)));+ // Two different values must not collide on one digest.+ expect(new Set(rows.map((row) => row.secretValueOrgBlindIndex)).size).toBe(2);+ }));// Version rows written before the org digest existed carry none, and a rollback copies a version's// digests straight onto the live secret. Without repair that silently puts an unindexed row back@@ -313,17 +328,16 @@ describe("Org-wide secret value tracking", () => {expect(outsider.json().secrets).toEqual([]);});- test("a second enable while a run is moving is refused", async () => {- await makeOrgIncomplete(orgId);+ test("a second enable while a run is moving does not start a second walk", async () =>+ withOwnOrg("org-value-tracking-double", async (ctx) => {+ await makeOrgIncomplete(ctx.orgId);- expect((await enable(authToken)).statusCode).toBe(200);+ expect((await enable(ctx.authToken)).statusCode).toBe(200);+ // Accepted rather than refused: one job id per scope means the second enable is absorbed by+ // the run already in flight instead of starting a competing one.+ expect((await enable(ctx.authToken)).statusCode).toBe(200);- const second = await enable(authToken);- // Either the first run is still moving, so the guard refuses, or it already finished, in which- // case the flag refuses instead. Both are a 400 and both are correct.- expect(second.statusCode).toBe(400);- expect(second.json().message).toMatch(/already (running|enabled)/i);-- await waitForCompletion(authToken);- });+ await waitForCompletion(ctx.authToken);+ expect((await status(ctx.authToken)).status).toBe("completed");+ }));});diff --git a/backend/src/keystore/keystore.ts b/backend/src/keystore/keystore.tsindex 4cb28cf4d8..5b295b7bfe 100644--- a/backend/src/keystore/keystore.ts+++ b/backend/src/keystore/keystore.ts@@ -179,10 +179,6 @@ export const KeyStorePrefixes = {// UUIDs and the endpoint segments do not overlap, so one prefix serves both without collision.InsightsCache: (scopeId: string, endpoint: string) => `insights-cache:${scopeId}:${endpoint}` as const,- // Holds a secret value tracking backfill's transient state. Completion is the durable flag on the- // organization, so nothing here needs to outlive the run.- SecretValueTrackingBackfill: (scopeId: string) => `secret-value-tracking-backfill:${scopeId}` as const,-// Braces are a Redis Cluster hash tag: the index zset and every member's payload key must land on// one slot for the multi-key upsert/delete scripts.WorkerHeartbeatIndex: (workerType: string) => `worker-heartbeat:{${workerType}}` as const,@@ -252,7 +248,6 @@ export const KeyStoreTtls = {EmailDispatchCooldownInSeconds: 60, // 1 minuteEmailDispatchMailboxWindowInSeconds: 86400, // 24 hoursEmailDispatchAbuseProbeInSeconds: 7200, // 2 hours- SecretValueTrackingBackfillInSeconds: 172800, // 2 days, long enough to resume a run someone retries the next dayInsightsCacheInSeconds: 300, // 5 minutesInsightsDuplicationCacheInSeconds: 3600, // 1 hourInsightsWeeklyHistoryCacheInSeconds: 86400, // 24 hoursdiff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.tsindex d583bc625b..b7c7e8f79b 100644--- a/backend/src/server/routes/index.ts+++ b/backend/src/server/routes/index.ts@@ -2338,7 +2338,6 @@ export const registerRoutes = async (permissionService,orgDAL,projectDAL,- keyStore,secretValueTrackingQueue});diff --git a/backend/src/services/secret-value-tracking/secret-value-tracking-fns.test.ts b/backend/src/services/secret-value-tracking/secret-value-tracking-fns.test.tsindex 95d55efb6b..b20c0b4654 100644--- a/backend/src/services/secret-value-tracking/secret-value-tracking-fns.test.ts+++ b/backend/src/services/secret-value-tracking/secret-value-tracking-fns.test.ts@@ -1,7 +1,4 @@-import { JobState } from "@app/queue/queue-service";--import { advanceCursor, isUsableRunState, needsBackfill, resolveRunStatus } from "./secret-value-tracking-fns";-import { TBackfillRunState } from "./secret-value-tracking-types";+import { advanceCursor, needsBackfill } from "./secret-value-tracking-fns";const PROJECTS = ["p1", "p2"];const FOLDERS = { p1: ["f1", "f2"], p2: ["f3"] };@@ -166,94 +163,3 @@ describe("needsBackfill", () => {);});});--const NOW = new Date("2026-09-24T12:00:00.000Z");-const minutesAgo = (n: number) => new Date(NOW.getTime() - n * 60_000).toISOString();--const running = (lastProgressAt: string): TBackfillRunState => ({- status: "running",- cursor: null,- projectsTotal: 3,- projectsDone: 1,- secretsProcessed: 10,- lastProgressAt-});--describe("resolveRunStatus", () => {- test("the flag being set means completed, whatever the key says", () => {- expect(resolveRunStatus(running(minutesAgo(1)), true, NOW)).toEqual({ status: JobState.Completed });- });-- test("no key and no flag means the backfill was never run", () => {- expect(resolveRunStatus(null, false, NOW)).toEqual({ status: JobState.NotFound });- });-- test("a run that progressed recently is pending", () => {- expect(resolveRunStatus(running(minutesAgo(1)), false, NOW)).toEqual({ status: JobState.Pending });- });-- test("a recorded failure reports its error", () => {- expect(- resolveRunStatus({ ...running(minutesAgo(1)), status: "failed", error: "kms unavailable" }, false, NOW)- ).toEqual({ status: JobState.Failed, message: "kms unavailable" });- });-- test("a run with no progress for longer than the window is stalled", () => {- const result = resolveRunStatus(running(minutesAgo(30)), false, NOW);- expect(result.status).toBe(JobState.Failed);- expect(result.message).toMatch(/stopped responding/i);- });-- // The boundary must be decisive rather than flapping between running and stalled.- test("exactly at the staleness boundary the run still counts as pending", () => {- expect(resolveRunStatus(running(minutesAgo(15)), false, NOW)).toEqual({ status: JobState.Pending });- });-- test("one second past the boundary the run is stalled", () => {- const justPast = new Date(NOW.getTime() - (15 * 60_000 + 1000)).toISOString();- expect(resolveRunStatus(running(justPast), false, NOW).status).toBe(JobState.Failed);- });-- // An unreadable timestamp must not throw into the status endpoint.- test("an unparseable lastProgressAt reads as stalled rather than throwing", () => {- expect(resolveRunStatus({ ...running(""), lastProgressAt: "not-a-date" }, false, NOW).status).toBe(JobState.Failed);- });-});--describe("run state shape validation", () => {- test("a payload with non-numeric counters is rejected rather than yielding NaN", () => {- // The status endpoint's response schema rejects NaN, so a payload like this would turn the one- // endpoint whose job is to degrade gracefully into a 500.- expect(isUsableRunState({ status: "running", lastProgressAt: NOW.toISOString(), projectsDone: "many" })).toBe(- false- );- });-- test("a well formed payload is usable", () => {- expect(isUsableRunState(running(minutesAgo(1)))).toBe(true);- });-- test("a payload missing its timestamp is rejected", () => {- expect(isUsableRunState({ status: "running", projectsTotal: 1, projectsDone: 0, secretsProcessed: 0 })).toBe(false);- });-});--describe("a finished run's state", () => {- test("a completed state is usable, so its counters survive to the status endpoint", () => {- expect(isUsableRunState({ ...running(minutesAgo(1)), status: "completed", cursor: null })).toBe(true);- });-- test("the flag stays the durable answer, and the state only supplies the numbers", () => {- const completed: TBackfillRunState = { ...running(minutesAgo(1)), status: "completed", cursor: null };- expect(resolveRunStatus(completed, true, NOW)).toEqual({ status: JobState.Completed });- });-});--describe("a completed state without the flag", () => {- // The flag is the truth. A leftover completed state whose flag is off means the work was undone- // or never landed, so it needs a run, not a spinner waiting for one that is not coming.- test("reports that no run is in progress rather than pending", () => {- const completed: TBackfillRunState = { ...running(minutesAgo(1)), status: "completed", cursor: null };- expect(resolveRunStatus(completed, false, NOW)).toEqual({ status: JobState.NotFound });- });-});diff --git a/backend/src/services/secret-value-tracking/secret-value-tracking-fns.ts b/backend/src/services/secret-value-tracking/secret-value-tracking-fns.tsindex 6eb6f07dd1..018ce5f86a 100644--- a/backend/src/services/secret-value-tracking/secret-value-tracking-fns.ts+++ b/backend/src/services/secret-value-tracking/secret-value-tracking-fns.ts@@ -1,13 +1,4 @@-import { JobState } from "@app/queue/queue-service";--import {- TAdvanceCursorInput,- TAdvanceCursorResult,- TBackfillCursor,- TBackfillRunState-} from "./secret-value-tracking-types";--export const BACKFILL_STALE_AFTER_MS = 15 * 60 * 1000;+import { TAdvanceCursorInput, TAdvanceCursorResult, TBackfillCursor } from "./secret-value-tracking-types";const startOfFolder = (projectId: string, folderId: string): TBackfillCursor => ({projectId,@@ -97,42 +88,3 @@ export const needsBackfill = (row: {secretValueOrgBlindIndex?: string | null;encryptedValue?: Buffer | null;}) => Boolean(row.encryptedValue) && (!row.secretValueBlindIndex || !row.secretValueOrgBlindIndex);--// Completion lives on the durable flag rather than in the run state, so a key that outlives a-// finished run can never contradict it.-export const resolveRunStatus = (- state: TBackfillRunState | null,- flagEnabled: boolean,- now: Date-): { status: JobState; message?: string } => {- if (flagEnabled) return { status: JobState.Completed };- if (!state) return { status: JobState.NotFound };- if (state.status === "failed") return { status: JobState.Failed, message: state.error ?? "Unknown error" };- // A completed state with the flag off is a run whose result was undone or never landed. It needs- // another run, so saying "pending" would leave a caller waiting for one that is not coming.- if (state.status === "completed") return { status: JobState.NotFound };-- const lastProgress = new Date(state.lastProgressAt).getTime();- if (Number.isNaN(lastProgress) || now.getTime() - lastProgress > BACKFILL_STALE_AFTER_MS) {- return {- status: JobState.Failed,- message: "The backfill stopped responding. Start it again to resume from where it left off."- };- }-- return { status: JobState.Pending };-};--// The status endpoint answers from this, so a payload it cannot use has to read as "no run in-// progress" rather than reach the response schema. The counters matter as much as the timestamp:-// a non-numeric one becomes NaN, which `z.number()` rejects, turning a graceful degrade into a 500.-export const isUsableRunState = (value: unknown): value is TBackfillRunState => {- const state = value as Partial<TBackfillRunState> | null;- if (!state || typeof state !== "object") return false;- if (state.status !== "running" && state.status !== "failed" && state.status !== "completed") return false;- if (typeof state.lastProgressAt !== "string") return false;-- return (["projectsTotal", "projectsDone", "secretsProcessed"] as const).every((field) =>- Number.isFinite(state[field])- );-};diff --git a/backend/src/services/secret-value-tracking/secret-value-tracking-queue.test.ts b/backend/src/services/secret-value-tracking/secret-value-tracking-queue.test.tsindex b9b5168c96..bb98b14dc1 100644--- a/backend/src/services/secret-value-tracking/secret-value-tracking-queue.test.ts+++ b/backend/src/services/secret-value-tracking/secret-value-tracking-queue.test.ts@@ -1,16 +1,14 @@-import { KeyStorePrefixes } from "@app/keystore/keystore";import { initLogger } from "@app/lib/logger";import { secretValueTrackingQueueFactory } from "./secret-value-tracking-queue";-import { TBackfillRunState } from "./secret-value-tracking-types";const ORG_ID = "org-1";type TFakeSecret = { id: string; key: string; folderId: string; hasOrgDigest: boolean };-// Drives the real job handler over in-memory fakes. The e2e suite cannot reach this layer: it runs-// one project whose secrets fit in a single chunk, so every bug that only shows on a resumed chunk-// or at a project boundary is invisible to it.+// Drives the real job handler over in-memory fakes. The e2e suite runs one project with a handful+// of secrets, so anything that only shows at a project boundary, or across many projects, is+// invisible to it.const makeHarness = ({projectIds,foldersByProject,@@ -20,37 +18,25 @@ const makeHarness = ({foldersByProject: Record<string, string[]>;secrets?: TFakeSecret[];}) => {- const store = new Map<string, string>();const queued: { payload: unknown; opts: Record<string, unknown> }[] = [];const flaggedProjects: string[] = [];const flaggedOrgs: string[] = [];const folderReads: string[] = [];+ const progress: { projectsTotal: number; projectsDone: number; secretsProcessed: number }[] = [];let rows = [...secrets];- let handler: (job: { data: unknown }) => Promise<void> = async () => {};+ let handler: (job: {+ data: unknown;+ updateProgress: (p: unknown) => Promise<void>;+ }) => Promise<void> = async () => {};- const keyStore = {- getItemPrimary: async (key: string) => store.get(key) ?? null,- setItemWithExpiry: async (key: string, _ttl: number | string, value: string | number | Buffer) => {- store.set(key, String(value));- return "OK" as const;- },- setItemWithExpiryNX: async (key: string, _ttl: number | string, value: string | number | Buffer) => {- if (store.has(key)) return null;- store.set(key, String(value));- return "OK" as const;- },- deleteItem: async (key: string) => {- store.delete(key);- return 1;- },- deleteItems: async () => 0- };+ const keyStore = { deleteItems: async () => 0 };const queueService = {- start: (_name: unknown, jobHandler: (job: { data: unknown }) => Promise<void>) => {+ start: (_name: unknown, jobHandler: typeof handler) => {handler = jobHandler;},listen: () => {},+ getJob: async () => undefined,queue: async (_name: unknown, _job: unknown, payload: unknown, opts: Record<string, unknown>) => {queued.push({ payload, opts });}@@ -106,141 +92,113 @@ const makeHarness = ({} as never});- const stateKey = KeyStorePrefixes.SecretValueTrackingBackfill(ORG_ID);-return {factory,queued,flaggedProjects,flaggedOrgs,folderReads,+ progress,rowsNow: () => rows,- setState: (state: TBackfillRunState) => store.set(stateKey, JSON.stringify(state)),- getState: () => {- const raw = store.get(stateKey);- return raw ? (JSON.parse(raw) as TBackfillRunState) : null;- },- runChunk: () => handler({ data: { scope: "org", orgId: ORG_ID } })+ run: () =>+ handler({+ data: { scope: "org", orgId: ORG_ID },+ updateProgress: async (p) => {+ progress.push(p as { projectsTotal: number; projectsDone: number; secretsProcessed: number });+ }+ })};};-const runningState = (cursor: TBackfillRunState["cursor"]): TBackfillRunState => ({- status: "running",- cursor,- projectsTotal: 2,- projectsDone: 0,- secretsProcessed: 0,- lastProgressAt: new Date().toISOString()-});-describe("the backfill job", () => {// The job logs its own lifecycle, and the unit environment boots no server to initialise it.beforeAll(() => {initLogger();});- test("a chunk resuming inside one project still reaches the org's other projects", async () => {+ test("one run covers every project in the org", async () => {const harness = makeHarness({- projectIds: ["p1", "p2"],- foldersByProject: { p1: ["f1"], p2: ["f2"] },+ projectIds: ["p1", "p2", "p3"],+ foldersByProject: { p1: ["f1"], p2: ["f2"], p3: ["f3"] },secrets: [- { id: "s1", key: "A", folderId: "f1", hasOrgDigest: true },- { id: "s2", key: "B", folderId: "f2", hasOrgDigest: false }+ { id: "s1", key: "A", folderId: "f1", hasOrgDigest: false },+ { id: "s2", key: "B", folderId: "f2", hasOrgDigest: false },+ { id: "s3", key: "C", folderId: "f3", hasOrgDigest: false }]});- // A resumed chunk: the cursor is already inside p1, exactly as the previous chunk left it.- harness.setState(runningState({ projectId: "p1", folderId: "f1", key: "A", id: "s1" }));+ await harness.run();- await harness.runChunk();-- // The bug this pins: the resumed chunk saw only p1's folders, read every later project as- // empty, stopped at the p1 boundary and flagged the org complete with s2 still unindexed.- expect(harness.rowsNow().find((row) => row.id === "s2")?.hasOrgDigest).toBe(true);- expect(harness.flaggedProjects).toEqual(["p1", "p2"]);- // Both projects really are done here, so flagging the org is right.+ expect(harness.rowsNow().every((row) => row.hasOrgDigest)).toBe(true);+ expect(harness.flaggedProjects).toEqual(["p1", "p2", "p3"]);expect(harness.flaggedOrgs).toEqual([ORG_ID]);});- test("a finished run leaves its final counters behind for the UI to show", async () => {+ test("a row that already carries both digests is left alone", async () => {const harness = makeHarness({- projectIds: ["p1", "p2"],- foldersByProject: { p1: ["f1"], p2: ["f2"] },+ projectIds: ["p1"],+ foldersByProject: { p1: ["f1"] },secrets: [- { id: "s1", key: "A", folderId: "f1", hasOrgDigest: false },- { id: "s2", key: "B", folderId: "f2", hasOrgDigest: false }+ { id: "done", key: "A", folderId: "f1", hasOrgDigest: true },+ { id: "todo", key: "B", folderId: "f1", hasOrgDigest: false }]});- harness.setState(runningState(null));- await harness.runChunk();+ await harness.run();- // The durable answer to "is it done" is still the flag. These numbers only let the UI show what- // the run got through, so they have to survive the run rather than be cleared with the cursor.- const finished = harness.getState();- expect(finished?.status).toBe("completed");- expect(finished?.projectsDone).toBe(2);- expect(finished?.secretsProcessed).toBe(2);- expect(finished?.cursor).toBeNull();+ // Only the one that needed work is counted, which is what makes a re-run cheap.+ expect(harness.progress.at(-1)?.secretsProcessed).toBe(1);});- test("a cursor on a project deleted since the last chunk carries on with the rest", async () => {+ test("a project that has been deleted does not stop the walk", async () => {const harness = makeHarness({projectIds: ["p1", "p2"],- foldersByProject: { p1: ["f1"], p2: ["f2"] },+ foldersByProject: { p1: [], p2: ["f2"] },secrets: [{ id: "s2", key: "B", folderId: "f2", hasOrgDigest: false }]});- harness.setState(runningState({ projectId: "gone", folderId: "gone-folder", key: "", id: "" }));+ await harness.run();- await harness.runChunk();-- expect(harness.flaggedProjects).not.toContain("gone");expect(harness.rowsNow().find((row) => row.id === "s2")?.hasOrgDigest).toBe(true);+ expect(harness.flaggedOrgs).toEqual([ORG_ID]);});- test("a project's folders are only read when the walk reaches it", async () => {+ test("a project with no folders at all is skipped rather than flagged", async () => {const harness = makeHarness({- projectIds: ["p1", "p2", "p3"],- foldersByProject: { p1: ["f1"], p2: ["f2"], p3: ["f3"] },- secrets: [{ id: "s1", key: "A", folderId: "f1", hasOrgDigest: false }]+ projectIds: ["empty", "p2"],+ foldersByProject: { empty: [], p2: ["f2"] },+ secrets: [{ id: "s2", key: "B", folderId: "f2", hasOrgDigest: false }]});- harness.setState(runningState({ projectId: "p1", folderId: "f1", key: "", id: "" }));- await harness.runChunk();+ await harness.run();- // Every project is reached in this run, but each is read once rather than all up front.- expect(harness.folderReads.filter((id) => id === "p1")).toHaveLength(1);+ expect(harness.flaggedProjects).toEqual(["p2"]);});- test("a run over many empty folders checkpoints instead of querying without bound", async () => {- const folders = Array.from({ length: 5000 }, (_, i) => `empty-${i}`);- const harness = makeHarness({- projectIds: ["p1"],- foldersByProject: { p1: folders },- secrets: []- });+ test("an org with nothing in it still finishes and is flagged", async () => {+ const harness = makeHarness({ projectIds: ["p1"], foldersByProject: { p1: ["f1"] }, secrets: [] });- harness.setState(runningState(null));- await harness.runChunk();+ await harness.run();- // Empty folders read no rows, so a chunk bounded only on rows would walk all 5000 in one job.- expect(harness.queued).toHaveLength(1);- expect(harness.getState()?.cursor).not.toBeNull();- // Work remains, so nothing may claim the org is searchable yet.- expect(harness.flaggedOrgs).toEqual([]);+ expect(harness.flaggedOrgs).toEqual([ORG_ID]);+ expect(harness.progress.at(-1)?.secretsProcessed).toBe(0);});- test("a queued chunk asks for retries so one transient failure does not kill the chain", async () => {+ test("progress is reported as the walk goes, not only at the end", async () => {const harness = makeHarness({- projectIds: ["p1"],- foldersByProject: { p1: Array.from({ length: 5000 }, (_, i) => `empty-${i}`) },- secrets: []+ projectIds: ["p1", "p2"],+ foldersByProject: { p1: ["f1"], p2: ["f2"] },+ secrets: [+ { id: "s1", key: "A", folderId: "f1", hasOrgDigest: false },+ { id: "s2", key: "B", folderId: "f2", hasOrgDigest: false }+ ]});- harness.setState(runningState(null));- await harness.runChunk();+ await harness.run();- expect(harness.queued[0].opts.attempts).toBeGreaterThan(1);+ // BullMQ reads these as the heartbeat that tells a working job from a stalled one, so a long+ // walk has to report more than once.+ expect(harness.progress.length).toBeGreaterThan(1);+ expect(harness.progress.at(-1)).toEqual({ projectsTotal: 2, projectsDone: 2, secretsProcessed: 2 });});});diff --git a/backend/src/services/secret-value-tracking/secret-value-tracking-queue.ts b/backend/src/services/secret-value-tracking/secret-value-tracking-queue.tsindex 6ec9c88131..6029b3b0d8 100644--- a/backend/src/services/secret-value-tracking/secret-value-tracking-queue.ts+++ b/backend/src/services/secret-value-tracking/secret-value-tracking-queue.ts@@ -2,6 +2,7 @@ import { ProjectType } from "@app/db/schemas";import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";import { logger } from "@app/lib/logger";import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";+import { JobState } from "@app/queue/queue-service";import { TKmsServiceFactory } from "../kms/kms-service";import { KmsDataKey } from "../kms/kms-types";@@ -11,28 +12,27 @@ import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";import { createOrgSecretBlindIndexer } from "../secret-v2-bridge/secret-blind-index-fns";import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal";import { advanceCursor, needsBackfill } from "./secret-value-tracking-fns";-import { secretValueTrackingStateFactory } from "./secret-value-tracking-state";import { TBackfillCursor, TBackfillScope } from "./secret-value-tracking-types";-// A chunk is many read batches rather than one, because the per-project data key is resolved once-// per chunk: on an org using external KMS that resolve is a network call, and a chunk per batch-// would pay it five times as often for nothing.-const CHUNK_SIZE = 5000;const READ_BATCH_SIZE = 1000;-// A folder that holds nothing still costs a query, so a chunk is bounded on steps as well as rows.-// Without it a project of tens of thousands of empty folders runs unbounded inside one job.-const MAX_STEPS_PER_CHUNK = 500;-// Refreshes lastProgressAt inside a long chunk. Left to the end of the chunk, a slow one reads as-// stalled and the enable guard lets a second chain take the same scope while this one is still-// walking it.-const HEARTBEAT_EVERY_STEPS = 50;+// Decrypting a batch and hashing it twice is CPU work on a single-threaded runtime, so the walk+// yields between batches. Without it a large scope holds the event loop for as long as it runs and+// every other request waits behind it.+const PAUSE_BETWEEN_BATCHES_MS = 100;++const NO_PROGRESS = { projectsTotal: 0, projectsDone: 0, secretsProcessed: 0 };++export type TBackfillState = {+ status: JobState;+ message?: string;+ projectsTotal: number;+ projectsDone: number;+ secretsProcessed: number;+};type TSecretValueTrackingQueueFactoryDep = {queueService: TQueueServiceFactory;- keyStore: Pick<- TKeyStoreFactory,- "getItemPrimary" | "setItemWithExpiry" | "setItemWithExpiryNX" | "deleteItem" | "deleteItems"- >;+ keyStore: Pick<TKeyStoreFactory, "deleteItems">;projectDAL: Pick<TProjectDALFactory, "find" | "findById" | "updateById">;orgDAL: Pick<TOrgDALFactory, "updateById">;folderDAL: Pick<TSecretFolderDALFactory, "findByProjectId">;@@ -40,10 +40,8 @@ type TSecretValueTrackingQueueFactoryDep = {kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;};-// A payload with no scope is one queued before the org-wide walk shipped. It is read as project-// scope so the shape is understood, but such a job has no run state and the handler stops on that:-// the old code kept no state, so there is no cursor to resume and nothing to safely assume. The-// enable button simply comes back for the customer to press again.+// A payload with no scope predates the org-wide walk. Reading it as project scope is what lets a job+// queued seconds before a deploy still do the thing it was queued to do.const resolveScope = (data: { scope?: "org" | "project"; orgId?: string; projectId?: string }): TBackfillScope => {if (data.scope === "org" && data.orgId) return { scope: "org", orgId: data.orgId };return { scope: "project", projectId: data.projectId as string };@@ -51,6 +49,10 @@ const resolveScope = (data: { scope?: "org" | "project"; orgId?: string; projectconst scopeIdOf = (scope: TBackfillScope) => (scope.scope === "org" ? scope.orgId : scope.projectId);+// One job id per scope. BullMQ will not add a second job under an id it already holds, and that is+// what stops two backfills running over the same scope.+const jobIdOf = (scope: TBackfillScope) => `secret-value-tracking-${scopeIdOf(scope)}`;+export const secretValueTrackingQueueFactory = ({queueService,keyStore,@@ -60,267 +62,190 @@ export const secretValueTrackingQueueFactory = ({secretV2BridgeDAL,kmsService}: TSecretValueTrackingQueueFactoryDep) => {- const state = secretValueTrackingStateFactory({ keyStore });+ const startBackfill = async (scope: TBackfillScope) => {+ const jobId = jobIdOf(scope);++ const existing = await queueService.getJob(QueueName.SecretBlindIndexMigration, jobId);+ if (existing) {+ const state = await existing.getState();+ // A finished record is kept only so a status poll can still see it. It is not a run, so it+ // must not stand in the way of starting one.+ if (state === JobState.Completed || state === JobState.Failed) await existing.remove();+ else return;+ }- // Each chunk needs its own id: BullMQ ignores an add for an id that is currently active, so a- // fixed id would break the chain at the second chunk.- const queueChunk = async (scope: TBackfillScope) =>- queueService.queue(QueueName.SecretBlindIndexMigration, QueueJobs.SecretBlindIndexMigration, scope, {+ await queueService.queue(QueueName.SecretBlindIndexMigration, QueueJobs.SecretBlindIndexMigration, scope, {+ // Brief, because completion itself is the durable flag; this only has to outlive a poll.removeOnComplete: { age: 60 },+ // A day, so someone who comes back the next morning can still read why it failed.removeOnFail: { age: 24 * 3600 },- // Without retries one transient database or KMS blip ends the chain and the customer has to- // start the backfill again by hand.attempts: 3,backoff: { type: "exponential", delay: 5000 },- jobId: `secret-value-tracking-${scopeIdOf(scope)}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`+ jobId});-- const $resolveProjectIds = async (scope: TBackfillScope) => {- if (scope.scope === "project") {- const project = await projectDAL.findById(scope.projectId);- return { orgId: project?.orgId ?? null, projectIds: project ? [project.id] : [] };- }-- const projects = await projectDAL.find({ orgId: scope.orgId, type: ProjectType.SecretManager });- return {- orgId: scope.orgId,- projectIds: projects.map((project) => project.id).sort()- };};- const $markProjectComplete = async (projectId: string) => {- await projectDAL.updateById(projectId, { secretBlindIndexEnabled: true });- await keyStore.deleteItems({ pattern: `${KeyStorePrefixes.InsightsCache(projectId, "secrets-duplication")}*` });- };+ const getBackfillState = async (scope: TBackfillScope): Promise<TBackfillState> => {+ const job = await queueService.getJob(QueueName.SecretBlindIndexMigration, jobIdOf(scope));+ if (!job) return { status: JobState.NotFound, ...NO_PROGRESS };- const handler = async (job: { data: { scope?: "org" | "project"; orgId?: string; projectId?: string } }) => {- const scope = resolveScope(job.data);- const scopeId = scopeIdOf(scope);+ // The job reports its own progress as it walks, so there is nowhere else to keep it.+ const reported = (job.progress ?? {}) as Partial<typeof NO_PROGRESS>;+ const progress = { ...NO_PROGRESS, ...reported };- const runState = await state.read(scopeId);- if (!runState) {- logger.info(`SecretValueTrackingBackfill: no run state, nothing to resume [scopeId=${scopeId}]`);- return;+ const state = await job.getState();+ if (state === JobState.Failed) {+ return { status: JobState.Failed, message: job.failedReason ?? "Unknown error", ...progress };}+ if (state === JobState.Completed) return { status: JobState.Completed, ...progress };+ return { status: JobState.Pending, ...progress };+ };- // A chunk whose run has since been recorded failed or completed belongs to a chain that has been- // superseded. Carrying on would walk a cursor another chain now owns, and a finished run's null- // cursor would restart the whole scope.- if (runState.status !== "running") {- logger.info(- `SecretValueTrackingBackfill: run is ${runState.status}, dropping superseded chunk [scopeId=${scopeId}]`- );- return;- }+ queueService.start(+ QueueName.SecretBlindIndexMigration,+ async (job) => {+ const scope = resolveScope(job.data);+ const scopeId = scopeIdOf(scope);- const { orgId, projectIds } = await $resolveProjectIds(scope);- if (!orgId) {- logger.info(`SecretValueTrackingBackfill: scope no longer exists [scopeId=${scopeId}]`);- await state.clear(scopeId);- return;- }+ const projects =+ scope.scope === "project"+ ? [await projectDAL.findById(scope.projectId)].filter(Boolean)+ : await projectDAL.find({ orgId: scope.orgId, type: ProjectType.SecretManager });- // One org data key for the whole chunk, since every project in the org shares it.- const { generateOrgLevelBlindIndex } = await createOrgSecretBlindIndexer({ orgId, kmsService });- const projectCiphers = new Map<- string,- {- decryptor: (input: { cipherTextBlob: Buffer }) => Buffer;- generateBlindIndex: (value: Buffer) => Promise<string>;+ if (!projects.length) {+ logger.info(`SecretValueTrackingBackfill: nothing to walk [scopeId=${scopeId}]`);+ return;}- >();- const foldersByProject: Record<string, string[]> = {};- const $foldersOf = async (projectId: string) => {- if (!foldersByProject[projectId]) {+ const { orgId } = projects[0];+ const projectIds = projects.map((project) => project.id).sort();++ // Every project's folders up front. The walk is one pass over the whole scope, so deferring+ // them buys nothing, and a complete map is what lets advanceCursor tell a project that has no+ // folders from one it has simply not been shown.+ const foldersByProject: Record<string, string[]> = {};+ for await (const projectId of projectIds) {// Soft-deleted environments are included: the delete is reversible, so leaving their rows- // unindexed and then flagging the org complete breaks the moment one is restored.+ // unindexed and then flagging the scope complete breaks the moment one is restored.const folders = await folderDAL.findByProjectId(projectId, undefined, true);foldersByProject[projectId] = folders.map((folder) => folder.id).sort();}- return foldersByProject[projectId];- };-- const $cipherOf = async (projectId: string) => {- const cached = projectCiphers.get(projectId);- if (cached) return cached;-- const { decryptor, generateSecretBlindIndex } = await kmsService.createCipherPairWithDataKey({- type: KmsDataKey.SecretManager,- projectId- });- const pair = { decryptor, generateBlindIndex: generateSecretBlindIndex };- projectCiphers.set(projectId, pair);- return pair;- };-- let { cursor } = runState;- let { projectsDone } = runState;- let { secretsProcessed } = runState;- let readThisChunk = 0;- let steps = 0;- let done = false;-- const $writeProgress = async (at: TBackfillCursor | null) => {- await state.write(scopeId, {- status: "running",- cursor: at,- projectsTotal: runState.projectsTotal,- projectsDone,- secretsProcessed,- lastProgressAt: new Date().toISOString()- });- };-- // advanceCursor only sees the projects whose folders are loaded, and reads an unloaded one as a- // project with none, so it would declare the walk finished at the first project boundary. Load- // the next unseeded project and ask again before accepting the end.- const $advance = async (- from: TBackfillCursor | null,- lastRow: { key: string; id: string } | null,- folderExhausted: boolean- ) => {- for (;;) {- const next = advanceCursor({- cursor: from,- projectIds,- folderIdsByProject: foldersByProject,- lastRow,- folderExhausted++ // One org data key for the whole walk, since every project in the org shares it.+ const { generateOrgLevelBlindIndex } = await createOrgSecretBlindIndexer({ orgId, kmsService });+ const projectCiphers = new Map<+ string,+ {+ decryptor: (input: { cipherTextBlob: Buffer }) => Buffer;+ generateBlindIndex: (value: Buffer) => Promise<string>;+ }+ >();++ const $cipherOf = async (projectId: string) => {+ const cached = projectCiphers.get(projectId);+ if (cached) return cached;++ const { decryptor, generateSecretBlindIndex } = await kmsService.createCipherPairWithDataKey({+ type: KmsDataKey.SecretManager,+ projectId});- if (!next.done) return next;+ const pair = { decryptor, generateBlindIndex: generateSecretBlindIndex };+ projectCiphers.set(projectId, pair);+ return pair;+ };- const unseeded = projectIds.find((projectId) => !(projectId in foldersByProject));- if (!unseeded) return next;- // eslint-disable-next-line no-await-in-loop- await $foldersOf(unseeded);- }- };+ const $markProjectComplete = async (projectId: string) => {+ await projectDAL.updateById(projectId, { secretBlindIndexEnabled: true });+ await keyStore.deleteItems({ pattern: `${KeyStorePrefixes.InsightsCache(projectId, "secrets-duplication")}*` });+ };- if (!cursor) {- const seeded = await $advance(null, null, false);- if (seeded.done) done = true;- else cursor = seeded.cursor;- }+ const $advance = (+ from: TBackfillCursor | null,+ lastRow: { key: string; id: string } | null,+ folderExhausted: boolean+ ) => advanceCursor({ cursor: from, projectIds, folderIdsByProject: foldersByProject, lastRow, folderExhausted });- while (!done && cursor && readThisChunk < CHUNK_SIZE && steps < MAX_STEPS_PER_CHUNK) {- steps += 1;- // eslint-disable-next-line no-await-in-loop- await $foldersOf(cursor.projectId);+ let secretsProcessed = 0;+ let projectsDone = 0;- // eslint-disable-next-line no-await-in-loop- const rows = await secretV2BridgeDAL.findSecretsInFolderAfter(- cursor.folderId,- { key: cursor.key, id: cursor.id },- READ_BATCH_SIZE- );- readThisChunk += rows.length;+ const seeded = $advance(null, null, false);+ let cursor = seeded.done ? null : seeded.cursor;- const pending = rows.filter(needsBackfill);- if (pending.length) {+ while (cursor) {// eslint-disable-next-line no-await-in-loop- const { decryptor, generateBlindIndex } = await $cipherOf(cursor.projectId);- // eslint-disable-next-line no-await-in-loop- const updates = await Promise.all(- pending.map(async (row) => {- const value = decryptor({ cipherTextBlob: row.encryptedValue as Buffer });- const [secretValueBlindIndex, secretValueOrgBlindIndex] = await Promise.all([- generateBlindIndex(value),- generateOrgLevelBlindIndex(value)- ]);- return { id: row.id, secretValueBlindIndex, secretValueOrgBlindIndex };- })+ const rows = await secretV2BridgeDAL.findSecretsInFolderAfter(+ cursor.folderId,+ { key: cursor.key, id: cursor.id },+ READ_BATCH_SIZE);- // eslint-disable-next-line no-await-in-loop- await secretV2BridgeDAL.batchSetBlindIndexes(updates);- secretsProcessed += updates.length;- }- const lastRow = rows.length ? { key: rows[rows.length - 1].key, id: rows[rows.length - 1].id } : null;- // eslint-disable-next-line no-await-in-loop- const next = await $advance(cursor, lastRow, rows.length < READ_BATCH_SIZE);+ const pending = rows.filter(needsBackfill);+ if (pending.length) {+ // eslint-disable-next-line no-await-in-loop+ const { decryptor, generateBlindIndex } = await $cipherOf(cursor.projectId);+ // eslint-disable-next-line no-await-in-loop+ const updates = await Promise.all(+ pending.map(async (row) => {+ const value = decryptor({ cipherTextBlob: row.encryptedValue as Buffer });+ const [secretValueBlindIndex, secretValueOrgBlindIndex] = await Promise.all([+ generateBlindIndex(value),+ generateOrgLevelBlindIndex(value)+ ]);+ return { id: row.id, secretValueBlindIndex, secretValueOrgBlindIndex };+ })+ );+ // eslint-disable-next-line no-await-in-loop+ await secretV2BridgeDAL.batchSetBlindIndexes(updates);+ secretsProcessed += updates.length;+ }++ const lastRow = rows.length ? { key: rows[rows.length - 1].key, id: rows[rows.length - 1].id } : null;+ const next = $advance(cursor, lastRow, rows.length < READ_BATCH_SIZE);- if (next.done) {- // The walk ends inside the project it was working, so that project's own flag is set here- // rather than from a completedProjectId the `done` branch never carries. A cursor left on a- // project that has since been deleted is not a project to flag.- if (projectIds.includes(cursor.projectId)) {+ if (next.done) {+ // The walk ends inside the project it was working, so that project's own flag is set here+ // rather than from a completedProjectId the `done` branch never carries.// eslint-disable-next-line no-await-in-loopawait $markProjectComplete(cursor.projectId);projectsDone += 1;+ cursor = null;+ break;}- done = true;- break;- }- if (next.completedProjectId) {- // eslint-disable-next-line no-await-in-loop- await $markProjectComplete(next.completedProjectId);- projectsDone += 1;- }- cursor = next.cursor;+ if (next.completedProjectId) {+ // eslint-disable-next-line no-await-in-loop+ await $markProjectComplete(next.completedProjectId);+ projectsDone += 1;+ }+ cursor = next.cursor;- if (steps % HEARTBEAT_EVERY_STEPS === 0) {+ // Doubles as the heartbeat BullMQ reads to tell a working job from a stalled one.+ // eslint-disable-next-line no-await-in-loop+ await job.updateProgress({ projectsTotal: projectIds.length, projectsDone, secretsProcessed });// eslint-disable-next-line no-await-in-loop- await $writeProgress(cursor);+ await new Promise((resolve) => {+ setTimeout(resolve, PAUSE_BETWEEN_BATCHES_MS);+ });}- }- if (done) {if (scope.scope === "org") await orgDAL.updateById(scope.orgId, { orgWideSecretValueTrackingEnabled: true });- // Kept rather than cleared, so the UI can show what the run got through. The flag above is- // what actually answers "is it done", and this key is free to expire.- await state.write(scopeId, {- status: "completed",- cursor: null,- projectsTotal: runState.projectsTotal,- projectsDone,- secretsProcessed,- lastProgressAt: new Date().toISOString()- });++ await job.updateProgress({ projectsTotal: projectIds.length, projectsDone, secretsProcessed });logger.info(`SecretValueTrackingBackfill: complete [scopeId=${scopeId}] [projectsDone=${projectsDone}] [secretsProcessed=${secretsProcessed}]`);- return;- }-- await $writeProgress(cursor);- await queueChunk(scope);- };-- queueService.start(QueueName.SecretBlindIndexMigration, handler, {- concurrency: 1,- limiter: { max: 1, duration: 1000 }- });+ },+ // One walk at a time, so a second scope waits rather than competing for the same connection pool+ // and the same CPU.+ { concurrency: 1 }+ );queueService.listen(QueueName.SecretBlindIndexMigration, "failed", (job, err) => {- const scopeId = job?.data ? scopeIdOf(resolveScope(job.data)) : undefined;- logger.error(err, `SecretValueTrackingBackfill: chunk failed [scopeId=${scopeId}]`);- if (!scopeId) return;-- // BullMQ raises this on every failed attempt, not only the last. Recording a failure while a- // retry is still queued would let the enable guard hand the scope to a second chain, and the two- // would then share one cursor.- const attemptsAllowed = job?.opts?.attempts ?? 1;- if ((job?.attemptsMade ?? 0) < attemptsAllowed) return;-- // Recording the failure is what lets the status endpoint answer immediately, rather than making- // the user wait out the staleness window.- void state- .read(scopeId)- .then((existing) =>- existing ? state.write(scopeId, { ...existing, status: "failed", error: err.message }) : undefined- )- .catch((writeErr) =>- logger.error(writeErr, `SecretValueTrackingBackfill: could not record failure [scopeId=${scopeId}]`)- );+ const scopeId = job?.data ? scopeIdOf(resolveScope(job.data)) : "unknown";+ logger.error(err, `SecretValueTrackingBackfill: failed [scopeId=${scopeId}]`);});- const startBackfill = async (scope: TBackfillScope) => {- await queueChunk(scope);- };-- return { startBackfill };+ return { startBackfill, getBackfillState };};export type TSecretValueTrackingQueueFactory = ReturnType<typeof secretValueTrackingQueueFactory>;diff --git a/backend/src/services/secret-value-tracking/secret-value-tracking-service.ts b/backend/src/services/secret-value-tracking/secret-value-tracking-service.tsindex 5c2877bf9a..a78c5fae2d 100644--- a/backend/src/services/secret-value-tracking/secret-value-tracking-service.ts+++ b/backend/src/services/secret-value-tracking/secret-value-tracking-service.ts@@ -4,21 +4,18 @@ import { ActionProjectType, OrganizationActionScope, ProjectType } from "@app/dbimport { OrgPermissionActions, OrgPermissionSubjects } from "@app/ee/services/permission/org-permission";import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service-types";import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission";-import { TKeyStoreFactory } from "@app/keystore/keystore";-import { BadRequestError, NotFoundError } from "@app/lib/errors";+import { NotFoundError } from "@app/lib/errors";import { OrgServiceActor, TProjectPermission } from "@app/lib/types";+import { JobState } from "@app/queue/queue-service";import { TOrgDALFactory } from "../org/org-dal";import { TProjectDALFactory } from "../project/project-dal";-import { resolveRunStatus } from "./secret-value-tracking-fns";import { TSecretValueTrackingQueueFactory } from "./secret-value-tracking-queue";-import { secretValueTrackingStateFactory } from "./secret-value-tracking-state";type TSecretValueTrackingServiceFactoryDep = {permissionService: Pick<TPermissionServiceFactory, "getOrgPermission" | "getProjectPermission">;orgDAL: Pick<TOrgDALFactory, "findById">;projectDAL: Pick<TProjectDALFactory, "find" | "findById">;- keyStore: Pick<TKeyStoreFactory, "getItemPrimary" | "setItemWithExpiry" | "setItemWithExpiryNX" | "deleteItem">;secretValueTrackingQueue: TSecretValueTrackingQueueFactory;};@@ -26,17 +23,8 @@ export const secretValueTrackingServiceFactory = ({permissionService,orgDAL,projectDAL,- keyStore,secretValueTrackingQueue}: TSecretValueTrackingServiceFactoryDep) => {- const state = secretValueTrackingStateFactory({ keyStore });-- const $progressOf = (runState: Awaited<ReturnType<typeof state.read>>) => ({- projectsTotal: runState?.projectsTotal ?? 0,- projectsDone: runState?.projectsDone ?? 0,- secretsProcessed: runState?.secretsProcessed ?? 0- });-const enableForOrg = async (actor: OrgServiceActor) => {const { permission } = await permissionService.getOrgPermission({actor: actor.type,@@ -55,30 +43,14 @@ export const secretValueTrackingServiceFactory = ({// unindexed row back into a completed org (a rollback to a version predating the digest, an// environment restored after the walk passed it), and without a re-run the only repair is SQL.// The walk skips rows that already carry both digests, so a redundant run costs a read pass.-- // Projects are counted so progress can be reported as "project M of N". Secrets deliberately are- // not: counting them is expensive on exactly the organizations where progress matters, and the- // number is stale the moment it is taken.+ //+ // A run that is already moving is refused by the queue instead: one job id per scope, and BullMQ+ // will not add a second job under an id it already holds.const projects = await projectDAL.find({ orgId: actor.orgId, type: ProjectType.SecretManager });- const projectsTotal = projects.length;-- const claimed = await state.claim(actor.orgId, projectsTotal);- if (!claimed) {- throw new BadRequestError({- message: "A secret value tracking backfill is already running for this organization"- });- }-- // Releasing the claim matters: without it a failed enqueue leaves the organization holding a- // claim it is not using, and the guard refuses every retry until the staleness window passes.- try {- await secretValueTrackingQueue.startBackfill({ scope: "org", orgId: actor.orgId });- } catch (error) {- await state.clear(actor.orgId);- throw error;- }-- return { projectsTotal };++ await secretValueTrackingQueue.startBackfill({ scope: "org", orgId: actor.orgId });++ return { projectsTotal: projects.length };};const getOrgStatus = async (actor: OrgServiceActor) => {@@ -95,11 +67,10 @@ export const secretValueTrackingServiceFactory = ({const org = await orgDAL.findById(actor.orgId);if (!org) throw new NotFoundError({ message: `Organization with ID '${actor.orgId}' not found` });- const runState = await state.read(actor.orgId);- return {- ...resolveRunStatus(runState, Boolean(org.orgWideSecretValueTrackingEnabled), new Date()),- ...$progressOf(runState)- };+ // The flag is the durable answer. The queue only says whether a run is in flight, or why the+ // last one stopped, and it drops a finished job within the minute.+ const progress = await secretValueTrackingQueue.getBackfillState({ scope: "org", orgId: actor.orgId });+ return org.orgWideSecretValueTrackingEnabled ? { ...progress, status: JobState.Completed } : progress;};const $assertProjectSettingsEdit = async ({@@ -127,28 +98,14 @@ export const secretValueTrackingServiceFactory = ({const enableForProject = async (dto: TProjectPermission) => {const project = await $assertProjectSettingsEdit(dto);-- const claimed = await state.claim(project.id, 1);- if (!claimed) {- throw new BadRequestError({ message: "A secret value tracking backfill is already running for this project" });- }-- try {- await secretValueTrackingQueue.startBackfill({ scope: "project", projectId: project.id });- } catch (error) {- await state.clear(project.id);- throw error;- }+ await secretValueTrackingQueue.startBackfill({ scope: "project", projectId: project.id });};const getProjectStatus = async (dto: TProjectPermission) => {const project = await $assertProjectSettingsEdit(dto);- const runState = await state.read(project.id);- return {- ...resolveRunStatus(runState, Boolean(project.secretBlindIndexEnabled), new Date()),- ...$progressOf(runState)- };+ const progress = await secretValueTrackingQueue.getBackfillState({ scope: "project", projectId: project.id });+ return project.secretBlindIndexEnabled ? { ...progress, status: JobState.Completed } : progress;};return { enableForOrg, getOrgStatus, enableForProject, getProjectStatus };diff --git a/backend/src/services/secret-value-tracking/secret-value-tracking-state.ts b/backend/src/services/secret-value-tracking/secret-value-tracking-state.tsdeleted file mode 100644index a58ea34e18..0000000000--- a/backend/src/services/secret-value-tracking/secret-value-tracking-state.ts+++ /dev/null@@ -1,87 +0,0 @@-import { KeyStorePrefixes, KeyStoreTtls, TKeyStoreFactory } from "@app/keystore/keystore";-import { logger } from "@app/lib/logger";--import { BACKFILL_STALE_AFTER_MS, isUsableRunState } from "./secret-value-tracking-fns";-import { TBackfillRunState } from "./secret-value-tracking-types";--type TSecretValueTrackingStateFactoryDep = {- keyStore: Pick<TKeyStoreFactory, "getItemPrimary" | "setItemWithExpiry" | "setItemWithExpiryNX" | "deleteItem">;-};--export const secretValueTrackingStateFactory = ({ keyStore }: TSecretValueTrackingStateFactoryDep) => {- // Reads the primary rather than a replica: a chunk writes its cursor immediately before queueing- // its successor, so a replica read can hand the next chunk the cursor it already worked through.- const read = async (scopeId: string): Promise<TBackfillRunState | null> => {- const raw = await keyStore.getItemPrimary(KeyStorePrefixes.SecretValueTrackingBackfill(scopeId));- if (!raw) return null;-- try {- // An unreadable key means no run in progress, which the status endpoint can answer, rather- // than an exception it cannot.- const parsed: unknown = JSON.parse(raw);- return isUsableRunState(parsed) ? parsed : null;- } catch (error) {- logger.warn(error, `Secret value tracking backfill state unreadable [scopeId=${scopeId}]`);- return null;- }- };-- const write = async (scopeId: string, state: TBackfillRunState) => {- await keyStore.setItemWithExpiry(- KeyStorePrefixes.SecretValueTrackingBackfill(scopeId),- KeyStoreTtls.SecretValueTrackingBackfillInSeconds,- JSON.stringify(state)- );- };-- const clear = async (scopeId: string) => {- await keyStore.deleteItem(KeyStorePrefixes.SecretValueTrackingBackfill(scopeId));- };-- const isStalled = (state: TBackfillRunState) => {- // A finished run holds the key only to keep its counters readable, so it never blocks a new one.- if (state.status === "failed" || state.status === "completed") return true;- const lastProgress = new Date(state.lastProgressAt).getTime();- return Number.isNaN(lastProgress) || Date.now() - lastProgress > BACKFILL_STALE_AFTER_MS;- };-- // Taking over a stalled run keeps its cursor, so retrying a backfill that died hours ago resumes- // instead of walking the whole scope again.- const claim = async (scopeId: string, projectsTotal: number) => {- const fresh: TBackfillRunState = {- status: "running",- cursor: null,- projectsTotal,- projectsDone: 0,- secretsProcessed: 0,- lastProgressAt: new Date().toISOString()- };-- const claimed = await keyStore.setItemWithExpiryNX(- KeyStorePrefixes.SecretValueTrackingBackfill(scopeId),- KeyStoreTtls.SecretValueTrackingBackfillInSeconds,- JSON.stringify(fresh)- );- if (claimed) return true;-- const existing = await read(scopeId);- if (!existing) {- await write(scopeId, fresh);- return true;- }-- if (!isStalled(existing)) return false;-- await write(scopeId, {- ...fresh,- cursor: existing.cursor,- projectsDone: existing.projectsDone,- secretsProcessed: existing.secretsProcessed- });- return true;- };-- return { read, write, clear, claim };-};--export type TSecretValueTrackingStateFactory = ReturnType<typeof secretValueTrackingStateFactory>;diff --git a/backend/src/services/secret-value-tracking/secret-value-tracking-types.ts b/backend/src/services/secret-value-tracking/secret-value-tracking-types.tsindex 622303deff..495955e39a 100644--- a/backend/src/services/secret-value-tracking/secret-value-tracking-types.ts+++ b/backend/src/services/secret-value-tracking/secret-value-tracking-types.ts@@ -7,18 +7,6 @@ export type TBackfillCursor = {export type TBackfillScope = { scope: "org"; orgId: string } | { scope: "project"; projectId: string };-export type TBackfillRunState = {- // "completed" carries the final counters so the UI can show what the run got through. It is never- // the answer to "is it done": that is the durable flag, and this key is allowed to expire.- status: "running" | "failed" | "completed";- cursor: TBackfillCursor | null;- projectsTotal: number;- projectsDone: number;- secretsProcessed: number;- lastProgressAt: string;- error?: string;-};-export type TAdvanceCursorInput = {cursor: TBackfillCursor | null;projectIds: string[];