Skip to content

Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.

refactor(secret-value-tracking): run the backfill as one job

Calvinb8e7bfec5809
diff --git a/backend/e2e-test/routes/v1/org-secret-value-tracking.spec.ts b/backend/e2e-test/routes/v1/org-secret-value-tracking.spec.ts
index e4364133a9..a5dbd9f623 100644
--- a/backend/e2e-test/routes/v1/org-secret-value-tracking.spec.ts
+++ b/backend/e2e-test/routes/v1/org-secret-value-tracking.spec.ts
@@ -93,6 +93,21 @@ const waitForCompletion = async (authToken: string, timeoutMs = 30_000) => {
}
};
+// A backfill is refused while one is already in flight for the same scope, so a test that drives a
+// run to completion cannot share an organization with the next one: the previous job may still be
+// finishing when the next enable arrives, and would swallow it. Each such test gets its own.
+const withOwnOrg = async (
+ name: string,
+ body: (ctx: Awaited<ReturnType<typeof createIsolatedOrgAndProject>>) => Promise<void>
+) => {
+ const ctx = await createIsolatedOrgAndProject(name);
+ try {
+ await body(ctx);
+ } finally {
+ await ctx.cleanup();
+ }
+};
+
describe("Org-wide secret value tracking", () => {
// The walk is a queue round trip per chunk, so it takes longer than the 5s default.
vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 });
@@ -116,77 +131,77 @@ describe("Org-wide secret value tracking", () => {
// Re-running a completed backfill is the repair path for every way an unindexed row can get back
// into a finished org, so it has to be accepted rather than refused as "already enabled".
- test("a completed organization can run the backfill again", async () => {
- expect((await enable(authToken)).statusCode).toBe(200);
- await waitForCompletion(authToken);
- expect((await status(authToken)).status).toBe("completed");
- });
-
- test("searching by value refuses while tracking is off, and works once the backfill completes", async () => {
- await createSecretV2({
- workspaceId: projectId,
- environmentSlug: ENV,
- secretPath: "/",
- key: "PREDATES_TRACKING",
- value: "written-before-tracking",
- authToken
- });
-
- await makeOrgIncomplete(orgId);
-
- const refused = await searchByValue("written-before-tracking", authToken);
- expect(refused.statusCode).toBe(400);
- expect(refused.json().message).toMatch(/org-wide secret value tracking/i);
-
- expect((await status(authToken)).status).toBe("not-found");
-
- const started = await enable(authToken);
- expect(started.statusCode).toBe(200);
-
- const finished = await waitForCompletion(authToken);
- expect(finished.projectsTotal).toBeGreaterThanOrEqual(1);
- expect(finished.projectsDone).toBeGreaterThanOrEqual(1);
-
- const found = await searchByValue("written-before-tracking", authToken);
- expect(found.statusCode).toBe(200);
- expect(found.json().secrets.map((s: { key: string }) => s.key)).toEqual(["PREDATES_TRACKING"]);
- });
+ test("a completed organization can run the backfill again", async () =>
+ withOwnOrg("org-value-tracking-rerun", async (ctx) => {
+ expect((await enable(ctx.authToken)).statusCode).toBe(200);
+ await waitForCompletion(ctx.authToken);
+ expect((await status(ctx.authToken)).status).toBe("completed");
+ }));
+
+ test("searching by value refuses while tracking is off, and works once the backfill completes", async () =>
+ withOwnOrg("org-value-tracking-search", async (ctx) => {
+ await createSecretV2({
+ workspaceId: ctx.projectId,
+ environmentSlug: ENV,
+ secretPath: "/",
+ key: "PREDATES_TRACKING",
+ value: "written-before-tracking",
+ authToken: ctx.authToken
+ });
+
+ await makeOrgIncomplete(ctx.orgId);
+
+ const refused = await searchByValue("written-before-tracking", ctx.authToken);
+ expect(refused.statusCode).toBe(400);
+ expect(refused.json().message).toMatch(/org-wide secret value tracking/i);
+
+ expect((await enable(ctx.authToken)).statusCode).toBe(200);
+
+ const finished = await waitForCompletion(ctx.authToken);
+ expect(finished.projectsTotal).toBeGreaterThanOrEqual(1);
+ expect(finished.projectsDone).toBeGreaterThanOrEqual(1);
+
+ const found = await searchByValue("written-before-tracking", ctx.authToken);
+ expect(found.statusCode).toBe(200);
+ expect(found.json().secrets.map((secret: { key: string }) => secret.key)).toEqual(["PREDATES_TRACKING"]);
+ }));
// The (key, id) cursor exists for this: the unique index on (key, folderId) is partial on
- // type = 'shared', so a personal override shares a key with the shared secret in its folder and a
+ // type = 'shared', so a personal override shares a key with the shared secret beside it and a
// key-only cursor would walk past one of them.
- test("a personal override sharing a key with a shared secret is backfilled too", async () => {
- await createSecretV2({
- workspaceId: projectId,
- environmentSlug: ENV,
- secretPath: "/",
- key: "SHARED_AND_PERSONAL",
- value: "the-shared-value",
- authToken
- });
- await createSecretV2({
- workspaceId: projectId,
- environmentSlug: ENV,
- secretPath: "/",
- key: "SHARED_AND_PERSONAL",
- value: "the-personal-value",
- type: SecretType.Personal,
- authToken
- });
-
- await makeOrgIncomplete(orgId);
- expect((await enable(authToken)).statusCode).toBe(200);
- await waitForCompletion(authToken);
-
- const rows = await testDb(TableName.SecretV2)
- .where({ key: "SHARED_AND_PERSONAL" })
- .select("id", "type", "secretValueOrgBlindIndex");
-
- expect(rows).toHaveLength(2);
- rows.forEach((row) => expect(row.secretValueOrgBlindIndex).toEqual(expect.any(String)));
- // Two different values must not collide on one digest.
- expect(new Set(rows.map((row) => row.secretValueOrgBlindIndex)).size).toBe(2);
- });
+ test("a personal override sharing a key with a shared secret is backfilled too", async () =>
+ withOwnOrg("org-value-tracking-personal", async (ctx) => {
+ await createSecretV2({
+ workspaceId: ctx.projectId,
+ environmentSlug: ENV,
+ secretPath: "/",
+ key: "SHARED_AND_PERSONAL",
+ value: "the-shared-value",
+ authToken: ctx.authToken
+ });
+ await createSecretV2({
+ workspaceId: ctx.projectId,
+ environmentSlug: ENV,
+ secretPath: "/",
+ key: "SHARED_AND_PERSONAL",
+ value: "the-personal-value",
+ type: SecretType.Personal,
+ authToken: ctx.authToken
+ });
+
+ await makeOrgIncomplete(ctx.orgId);
+ expect((await enable(ctx.authToken)).statusCode).toBe(200);
+ await waitForCompletion(ctx.authToken);
+
+ const rows = await testDb(TableName.SecretV2)
+ .where({ key: "SHARED_AND_PERSONAL" })
+ .select("id", "type", "secretValueOrgBlindIndex");
+
+ expect(rows).toHaveLength(2);
+ rows.forEach((row) => expect(row.secretValueOrgBlindIndex).toEqual(expect.any(String)));
+ // Two different values must not collide on one digest.
+ expect(new Set(rows.map((row) => row.secretValueOrgBlindIndex)).size).toBe(2);
+ }));
// Version rows written before the org digest existed carry none, and a rollback copies a version's
// digests straight onto the live secret. Without repair that silently puts an unindexed row back
@@ -313,17 +328,16 @@ describe("Org-wide secret value tracking", () => {
expect(outsider.json().secrets).toEqual([]);
});
- test("a second enable while a run is moving is refused", async () => {
- await makeOrgIncomplete(orgId);
+ test("a second enable while a run is moving does not start a second walk", async () =>
+ withOwnOrg("org-value-tracking-double", async (ctx) => {
+ await makeOrgIncomplete(ctx.orgId);
- expect((await enable(authToken)).statusCode).toBe(200);
+ expect((await enable(ctx.authToken)).statusCode).toBe(200);
+ // Accepted rather than refused: one job id per scope means the second enable is absorbed by
+ // the run already in flight instead of starting a competing one.
+ expect((await enable(ctx.authToken)).statusCode).toBe(200);
- const second = await enable(authToken);
- // Either the first run is still moving, so the guard refuses, or it already finished, in which
- // case the flag refuses instead. Both are a 400 and both are correct.
- expect(second.statusCode).toBe(400);
- expect(second.json().message).toMatch(/already (running|enabled)/i);
-
- await waitForCompletion(authToken);
- });
+ await waitForCompletion(ctx.authToken);
+ expect((await status(ctx.authToken)).status).toBe("completed");
+ }));
});
diff --git a/backend/src/keystore/keystore.ts b/backend/src/keystore/keystore.ts
index 4cb28cf4d8..5b295b7bfe 100644
--- a/backend/src/keystore/keystore.ts
+++ b/backend/src/keystore/keystore.ts
@@ -179,10 +179,6 @@ export const KeyStorePrefixes = {
// UUIDs and the endpoint segments do not overlap, so one prefix serves both without collision.
InsightsCache: (scopeId: string, endpoint: string) => `insights-cache:${scopeId}:${endpoint}` as const,
- // Holds a secret value tracking backfill's transient state. Completion is the durable flag on the
- // organization, so nothing here needs to outlive the run.
- SecretValueTrackingBackfill: (scopeId: string) => `secret-value-tracking-backfill:${scopeId}` as const,
-
// Braces are a Redis Cluster hash tag: the index zset and every member's payload key must land on
// one slot for the multi-key upsert/delete scripts.
WorkerHeartbeatIndex: (workerType: string) => `worker-heartbeat:{${workerType}}` as const,
@@ -252,7 +248,6 @@ export const KeyStoreTtls = {
EmailDispatchCooldownInSeconds: 60, // 1 minute
EmailDispatchMailboxWindowInSeconds: 86400, // 24 hours
EmailDispatchAbuseProbeInSeconds: 7200, // 2 hours
- SecretValueTrackingBackfillInSeconds: 172800, // 2 days, long enough to resume a run someone retries the next day
InsightsCacheInSeconds: 300, // 5 minutes
InsightsDuplicationCacheInSeconds: 3600, // 1 hour
InsightsWeeklyHistoryCacheInSeconds: 86400, // 24 hours
diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts
index d583bc625b..b7c7e8f79b 100644
--- a/backend/src/server/routes/index.ts
+++ b/backend/src/server/routes/index.ts
@@ -2338,7 +2338,6 @@ export const registerRoutes = async (
permissionService,
orgDAL,
projectDAL,
- keyStore,
secretValueTrackingQueue
});
diff --git a/backend/src/services/secret-value-tracking/secret-value-tracking-fns.test.ts b/backend/src/services/secret-value-tracking/secret-value-tracking-fns.test.ts
index 95d55efb6b..b20c0b4654 100644
--- a/backend/src/services/secret-value-tracking/secret-value-tracking-fns.test.ts
+++ b/backend/src/services/secret-value-tracking/secret-value-tracking-fns.test.ts
@@ -1,7 +1,4 @@
-import { JobState } from "@app/queue/queue-service";
-
-import { advanceCursor, isUsableRunState, needsBackfill, resolveRunStatus } from "./secret-value-tracking-fns";
-import { TBackfillRunState } from "./secret-value-tracking-types";
+import { advanceCursor, needsBackfill } from "./secret-value-tracking-fns";
const PROJECTS = ["p1", "p2"];
const FOLDERS = { p1: ["f1", "f2"], p2: ["f3"] };
@@ -166,94 +163,3 @@ describe("needsBackfill", () => {
);
});
});
-
-const NOW = new Date("2026-09-24T12:00:00.000Z");
-const minutesAgo = (n: number) => new Date(NOW.getTime() - n * 60_000).toISOString();
-
-const running = (lastProgressAt: string): TBackfillRunState => ({
- status: "running",
- cursor: null,
- projectsTotal: 3,
- projectsDone: 1,
- secretsProcessed: 10,
- lastProgressAt
-});
-
-describe("resolveRunStatus", () => {
- test("the flag being set means completed, whatever the key says", () => {
- expect(resolveRunStatus(running(minutesAgo(1)), true, NOW)).toEqual({ status: JobState.Completed });
- });
-
- test("no key and no flag means the backfill was never run", () => {
- expect(resolveRunStatus(null, false, NOW)).toEqual({ status: JobState.NotFound });
- });
-
- test("a run that progressed recently is pending", () => {
- expect(resolveRunStatus(running(minutesAgo(1)), false, NOW)).toEqual({ status: JobState.Pending });
- });
-
- test("a recorded failure reports its error", () => {
- expect(
- resolveRunStatus({ ...running(minutesAgo(1)), status: "failed", error: "kms unavailable" }, false, NOW)
- ).toEqual({ status: JobState.Failed, message: "kms unavailable" });
- });
-
- test("a run with no progress for longer than the window is stalled", () => {
- const result = resolveRunStatus(running(minutesAgo(30)), false, NOW);
- expect(result.status).toBe(JobState.Failed);
- expect(result.message).toMatch(/stopped responding/i);
- });
-
- // The boundary must be decisive rather than flapping between running and stalled.
- test("exactly at the staleness boundary the run still counts as pending", () => {
- expect(resolveRunStatus(running(minutesAgo(15)), false, NOW)).toEqual({ status: JobState.Pending });
- });
-
- test("one second past the boundary the run is stalled", () => {
- const justPast = new Date(NOW.getTime() - (15 * 60_000 + 1000)).toISOString();
- expect(resolveRunStatus(running(justPast), false, NOW).status).toBe(JobState.Failed);
- });
-
- // An unreadable timestamp must not throw into the status endpoint.
- test("an unparseable lastProgressAt reads as stalled rather than throwing", () => {
- expect(resolveRunStatus({ ...running(""), lastProgressAt: "not-a-date" }, false, NOW).status).toBe(JobState.Failed);
- });
-});
-
-describe("run state shape validation", () => {
- test("a payload with non-numeric counters is rejected rather than yielding NaN", () => {
- // The status endpoint's response schema rejects NaN, so a payload like this would turn the one
- // endpoint whose job is to degrade gracefully into a 500.
- expect(isUsableRunState({ status: "running", lastProgressAt: NOW.toISOString(), projectsDone: "many" })).toBe(
- false
- );
- });
-
- test("a well formed payload is usable", () => {
- expect(isUsableRunState(running(minutesAgo(1)))).toBe(true);
- });
-
- test("a payload missing its timestamp is rejected", () => {
- expect(isUsableRunState({ status: "running", projectsTotal: 1, projectsDone: 0, secretsProcessed: 0 })).toBe(false);
- });
-});
-
-describe("a finished run's state", () => {
- test("a completed state is usable, so its counters survive to the status endpoint", () => {
- expect(isUsableRunState({ ...running(minutesAgo(1)), status: "completed", cursor: null })).toBe(true);
- });
-
- test("the flag stays the durable answer, and the state only supplies the numbers", () => {
- const completed: TBackfillRunState = { ...running(minutesAgo(1)), status: "completed", cursor: null };
- expect(resolveRunStatus(completed, true, NOW)).toEqual({ status: JobState.Completed });
- });
-});
-
-describe("a completed state without the flag", () => {
- // The flag is the truth. A leftover completed state whose flag is off means the work was undone
- // or never landed, so it needs a run, not a spinner waiting for one that is not coming.
- test("reports that no run is in progress rather than pending", () => {
- const completed: TBackfillRunState = { ...running(minutesAgo(1)), status: "completed", cursor: null };
- expect(resolveRunStatus(completed, false, NOW)).toEqual({ status: JobState.NotFound });
- });
-});
diff --git a/backend/src/services/secret-value-tracking/secret-value-tracking-fns.ts b/backend/src/services/secret-value-tracking/secret-value-tracking-fns.ts
index 6eb6f07dd1..018ce5f86a 100644
--- a/backend/src/services/secret-value-tracking/secret-value-tracking-fns.ts
+++ b/backend/src/services/secret-value-tracking/secret-value-tracking-fns.ts
@@ -1,13 +1,4 @@
-import { JobState } from "@app/queue/queue-service";
-
-import {
- TAdvanceCursorInput,
- TAdvanceCursorResult,
- TBackfillCursor,
- TBackfillRunState
-} from "./secret-value-tracking-types";
-
-export const BACKFILL_STALE_AFTER_MS = 15 * 60 * 1000;
+import { TAdvanceCursorInput, TAdvanceCursorResult, TBackfillCursor } from "./secret-value-tracking-types";
const startOfFolder = (projectId: string, folderId: string): TBackfillCursor => ({
projectId,
@@ -97,42 +88,3 @@ export const needsBackfill = (row: {
secretValueOrgBlindIndex?: string | null;
encryptedValue?: Buffer | null;
}) => Boolean(row.encryptedValue) && (!row.secretValueBlindIndex || !row.secretValueOrgBlindIndex);
-
-// Completion lives on the durable flag rather than in the run state, so a key that outlives a
-// finished run can never contradict it.
-export const resolveRunStatus = (
- state: TBackfillRunState | null,
- flagEnabled: boolean,
- now: Date
-): { status: JobState; message?: string } => {
- if (flagEnabled) return { status: JobState.Completed };
- if (!state) return { status: JobState.NotFound };
- if (state.status === "failed") return { status: JobState.Failed, message: state.error ?? "Unknown error" };
- // A completed state with the flag off is a run whose result was undone or never landed. It needs
- // another run, so saying "pending" would leave a caller waiting for one that is not coming.
- if (state.status === "completed") return { status: JobState.NotFound };
-
- const lastProgress = new Date(state.lastProgressAt).getTime();
- if (Number.isNaN(lastProgress) || now.getTime() - lastProgress > BACKFILL_STALE_AFTER_MS) {
- return {
- status: JobState.Failed,
- message: "The backfill stopped responding. Start it again to resume from where it left off."
- };
- }
-
- return { status: JobState.Pending };
-};
-
-// The status endpoint answers from this, so a payload it cannot use has to read as "no run in
-// progress" rather than reach the response schema. The counters matter as much as the timestamp:
-// a non-numeric one becomes NaN, which `z.number()` rejects, turning a graceful degrade into a 500.
-export const isUsableRunState = (value: unknown): value is TBackfillRunState => {
- const state = value as Partial<TBackfillRunState> | null;
- if (!state || typeof state !== "object") return false;
- if (state.status !== "running" && state.status !== "failed" && state.status !== "completed") return false;
- if (typeof state.lastProgressAt !== "string") return false;
-
- return (["projectsTotal", "projectsDone", "secretsProcessed"] as const).every((field) =>
- Number.isFinite(state[field])
- );
-};
diff --git a/backend/src/services/secret-value-tracking/secret-value-tracking-queue.test.ts b/backend/src/services/secret-value-tracking/secret-value-tracking-queue.test.ts
index b9b5168c96..bb98b14dc1 100644
--- a/backend/src/services/secret-value-tracking/secret-value-tracking-queue.test.ts
+++ b/backend/src/services/secret-value-tracking/secret-value-tracking-queue.test.ts
@@ -1,16 +1,14 @@
-import { KeyStorePrefixes } from "@app/keystore/keystore";
import { initLogger } from "@app/lib/logger";
import { secretValueTrackingQueueFactory } from "./secret-value-tracking-queue";
-import { TBackfillRunState } from "./secret-value-tracking-types";
const ORG_ID = "org-1";
type TFakeSecret = { id: string; key: string; folderId: string; hasOrgDigest: boolean };
-// Drives the real job handler over in-memory fakes. The e2e suite cannot reach this layer: it runs
-// one project whose secrets fit in a single chunk, so every bug that only shows on a resumed chunk
-// or at a project boundary is invisible to it.
+// Drives the real job handler over in-memory fakes. The e2e suite runs one project with a handful
+// of secrets, so anything that only shows at a project boundary, or across many projects, is
+// invisible to it.
const makeHarness = ({
projectIds,
foldersByProject,
@@ -20,37 +18,25 @@ const makeHarness = ({
foldersByProject: Record<string, string[]>;
secrets?: TFakeSecret[];
}) => {
- const store = new Map<string, string>();
const queued: { payload: unknown; opts: Record<string, unknown> }[] = [];
const flaggedProjects: string[] = [];
const flaggedOrgs: string[] = [];
const folderReads: string[] = [];
+ const progress: { projectsTotal: number; projectsDone: number; secretsProcessed: number }[] = [];
let rows = [...secrets];
- let handler: (job: { data: unknown }) => Promise<void> = async () => {};
+ let handler: (job: {
+ data: unknown;
+ updateProgress: (p: unknown) => Promise<void>;
+ }) => Promise<void> = async () => {};
- const keyStore = {
- getItemPrimary: async (key: string) => store.get(key) ?? null,
- setItemWithExpiry: async (key: string, _ttl: number | string, value: string | number | Buffer) => {
- store.set(key, String(value));
- return "OK" as const;
- },
- setItemWithExpiryNX: async (key: string, _ttl: number | string, value: string | number | Buffer) => {
- if (store.has(key)) return null;
- store.set(key, String(value));
- return "OK" as const;
- },
- deleteItem: async (key: string) => {
- store.delete(key);
- return 1;
- },
- deleteItems: async () => 0
- };
+ const keyStore = { deleteItems: async () => 0 };
const queueService = {
- start: (_name: unknown, jobHandler: (job: { data: unknown }) => Promise<void>) => {
+ start: (_name: unknown, jobHandler: typeof handler) => {
handler = jobHandler;
},
listen: () => {},
+ getJob: async () => undefined,
queue: async (_name: unknown, _job: unknown, payload: unknown, opts: Record<string, unknown>) => {
queued.push({ payload, opts });
}
@@ -106,141 +92,113 @@ const makeHarness = ({
} as never
});
- const stateKey = KeyStorePrefixes.SecretValueTrackingBackfill(ORG_ID);
-
return {
factory,
queued,
flaggedProjects,
flaggedOrgs,
folderReads,
+ progress,
rowsNow: () => rows,
- setState: (state: TBackfillRunState) => store.set(stateKey, JSON.stringify(state)),
- getState: () => {
- const raw = store.get(stateKey);
- return raw ? (JSON.parse(raw) as TBackfillRunState) : null;
- },
- runChunk: () => handler({ data: { scope: "org", orgId: ORG_ID } })
+ run: () =>
+ handler({
+ data: { scope: "org", orgId: ORG_ID },
+ updateProgress: async (p) => {
+ progress.push(p as { projectsTotal: number; projectsDone: number; secretsProcessed: number });
+ }
+ })
};
};
-const runningState = (cursor: TBackfillRunState["cursor"]): TBackfillRunState => ({
- status: "running",
- cursor,
- projectsTotal: 2,
- projectsDone: 0,
- secretsProcessed: 0,
- lastProgressAt: new Date().toISOString()
-});
-
describe("the backfill job", () => {
// The job logs its own lifecycle, and the unit environment boots no server to initialise it.
beforeAll(() => {
initLogger();
});
- test("a chunk resuming inside one project still reaches the org's other projects", async () => {
+ test("one run covers every project in the org", async () => {
const harness = makeHarness({
- projectIds: ["p1", "p2"],
- foldersByProject: { p1: ["f1"], p2: ["f2"] },
+ projectIds: ["p1", "p2", "p3"],
+ foldersByProject: { p1: ["f1"], p2: ["f2"], p3: ["f3"] },
secrets: [
- { id: "s1", key: "A", folderId: "f1", hasOrgDigest: true },
- { id: "s2", key: "B", folderId: "f2", hasOrgDigest: false }
+ { id: "s1", key: "A", folderId: "f1", hasOrgDigest: false },
+ { id: "s2", key: "B", folderId: "f2", hasOrgDigest: false },
+ { id: "s3", key: "C", folderId: "f3", hasOrgDigest: false }
]
});
- // A resumed chunk: the cursor is already inside p1, exactly as the previous chunk left it.
- harness.setState(runningState({ projectId: "p1", folderId: "f1", key: "A", id: "s1" }));
+ await harness.run();
- await harness.runChunk();
-
- // The bug this pins: the resumed chunk saw only p1's folders, read every later project as
- // empty, stopped at the p1 boundary and flagged the org complete with s2 still unindexed.
- expect(harness.rowsNow().find((row) => row.id === "s2")?.hasOrgDigest).toBe(true);
- expect(harness.flaggedProjects).toEqual(["p1", "p2"]);
- // Both projects really are done here, so flagging the org is right.
+ expect(harness.rowsNow().every((row) => row.hasOrgDigest)).toBe(true);
+ expect(harness.flaggedProjects).toEqual(["p1", "p2", "p3"]);
expect(harness.flaggedOrgs).toEqual([ORG_ID]);
});
- test("a finished run leaves its final counters behind for the UI to show", async () => {
+ test("a row that already carries both digests is left alone", async () => {
const harness = makeHarness({
- projectIds: ["p1", "p2"],
- foldersByProject: { p1: ["f1"], p2: ["f2"] },
+ projectIds: ["p1"],
+ foldersByProject: { p1: ["f1"] },
secrets: [
- { id: "s1", key: "A", folderId: "f1", hasOrgDigest: false },
- { id: "s2", key: "B", folderId: "f2", hasOrgDigest: false }
+ { id: "done", key: "A", folderId: "f1", hasOrgDigest: true },
+ { id: "todo", key: "B", folderId: "f1", hasOrgDigest: false }
]
});
- harness.setState(runningState(null));
- await harness.runChunk();
+ await harness.run();
- // The durable answer to "is it done" is still the flag. These numbers only let the UI show what
- // the run got through, so they have to survive the run rather than be cleared with the cursor.
- const finished = harness.getState();
- expect(finished?.status).toBe("completed");
- expect(finished?.projectsDone).toBe(2);
- expect(finished?.secretsProcessed).toBe(2);
- expect(finished?.cursor).toBeNull();
+ // Only the one that needed work is counted, which is what makes a re-run cheap.
+ expect(harness.progress.at(-1)?.secretsProcessed).toBe(1);
});
- test("a cursor on a project deleted since the last chunk carries on with the rest", async () => {
+ test("a project that has been deleted does not stop the walk", async () => {
const harness = makeHarness({
projectIds: ["p1", "p2"],
- foldersByProject: { p1: ["f1"], p2: ["f2"] },
+ foldersByProject: { p1: [], p2: ["f2"] },
secrets: [{ id: "s2", key: "B", folderId: "f2", hasOrgDigest: false }]
});
- harness.setState(runningState({ projectId: "gone", folderId: "gone-folder", key: "", id: "" }));
+ await harness.run();
- await harness.runChunk();
-
- expect(harness.flaggedProjects).not.toContain("gone");
expect(harness.rowsNow().find((row) => row.id === "s2")?.hasOrgDigest).toBe(true);
+ expect(harness.flaggedOrgs).toEqual([ORG_ID]);
});
- test("a project's folders are only read when the walk reaches it", async () => {
+ test("a project with no folders at all is skipped rather than flagged", async () => {
const harness = makeHarness({
- projectIds: ["p1", "p2", "p3"],
- foldersByProject: { p1: ["f1"], p2: ["f2"], p3: ["f3"] },
- secrets: [{ id: "s1", key: "A", folderId: "f1", hasOrgDigest: false }]
+ projectIds: ["empty", "p2"],
+ foldersByProject: { empty: [], p2: ["f2"] },
+ secrets: [{ id: "s2", key: "B", folderId: "f2", hasOrgDigest: false }]
});
- harness.setState(runningState({ projectId: "p1", folderId: "f1", key: "", id: "" }));
- await harness.runChunk();
+ await harness.run();
- // Every project is reached in this run, but each is read once rather than all up front.
- expect(harness.folderReads.filter((id) => id === "p1")).toHaveLength(1);
+ expect(harness.flaggedProjects).toEqual(["p2"]);
});
- test("a run over many empty folders checkpoints instead of querying without bound", async () => {
- const folders = Array.from({ length: 5000 }, (_, i) => `empty-${i}`);
- const harness = makeHarness({
- projectIds: ["p1"],
- foldersByProject: { p1: folders },
- secrets: []
- });
+ test("an org with nothing in it still finishes and is flagged", async () => {
+ const harness = makeHarness({ projectIds: ["p1"], foldersByProject: { p1: ["f1"] }, secrets: [] });
- harness.setState(runningState(null));
- await harness.runChunk();
+ await harness.run();
- // Empty folders read no rows, so a chunk bounded only on rows would walk all 5000 in one job.
- expect(harness.queued).toHaveLength(1);
- expect(harness.getState()?.cursor).not.toBeNull();
- // Work remains, so nothing may claim the org is searchable yet.
- expect(harness.flaggedOrgs).toEqual([]);
+ expect(harness.flaggedOrgs).toEqual([ORG_ID]);
+ expect(harness.progress.at(-1)?.secretsProcessed).toBe(0);
});
- test("a queued chunk asks for retries so one transient failure does not kill the chain", async () => {
+ test("progress is reported as the walk goes, not only at the end", async () => {
const harness = makeHarness({
- projectIds: ["p1"],
- foldersByProject: { p1: Array.from({ length: 5000 }, (_, i) => `empty-${i}`) },
- secrets: []
+ projectIds: ["p1", "p2"],
+ foldersByProject: { p1: ["f1"], p2: ["f2"] },
+ secrets: [
+ { id: "s1", key: "A", folderId: "f1", hasOrgDigest: false },
+ { id: "s2", key: "B", folderId: "f2", hasOrgDigest: false }
+ ]
});
- harness.setState(runningState(null));
- await harness.runChunk();
+ await harness.run();
- expect(harness.queued[0].opts.attempts).toBeGreaterThan(1);
+ // BullMQ reads these as the heartbeat that tells a working job from a stalled one, so a long
+ // walk has to report more than once.
+ expect(harness.progress.length).toBeGreaterThan(1);
+ expect(harness.progress.at(-1)).toEqual({ projectsTotal: 2, projectsDone: 2, secretsProcessed: 2 });
});
});
diff --git a/backend/src/services/secret-value-tracking/secret-value-tracking-queue.ts b/backend/src/services/secret-value-tracking/secret-value-tracking-queue.ts
index 6ec9c88131..6029b3b0d8 100644
--- a/backend/src/services/secret-value-tracking/secret-value-tracking-queue.ts
+++ b/backend/src/services/secret-value-tracking/secret-value-tracking-queue.ts
@@ -2,6 +2,7 @@ import { ProjectType } from "@app/db/schemas";
import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";
import { logger } from "@app/lib/logger";
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
+import { JobState } from "@app/queue/queue-service";
import { TKmsServiceFactory } from "../kms/kms-service";
import { KmsDataKey } from "../kms/kms-types";
@@ -11,28 +12,27 @@ import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
import { createOrgSecretBlindIndexer } from "../secret-v2-bridge/secret-blind-index-fns";
import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal";
import { advanceCursor, needsBackfill } from "./secret-value-tracking-fns";
-import { secretValueTrackingStateFactory } from "./secret-value-tracking-state";
import { TBackfillCursor, TBackfillScope } from "./secret-value-tracking-types";
-// A chunk is many read batches rather than one, because the per-project data key is resolved once
-// per chunk: on an org using external KMS that resolve is a network call, and a chunk per batch
-// would pay it five times as often for nothing.
-const CHUNK_SIZE = 5000;
const READ_BATCH_SIZE = 1000;
-// A folder that holds nothing still costs a query, so a chunk is bounded on steps as well as rows.
-// Without it a project of tens of thousands of empty folders runs unbounded inside one job.
-const MAX_STEPS_PER_CHUNK = 500;
-// Refreshes lastProgressAt inside a long chunk. Left to the end of the chunk, a slow one reads as
-// stalled and the enable guard lets a second chain take the same scope while this one is still
-// walking it.
-const HEARTBEAT_EVERY_STEPS = 50;
+// Decrypting a batch and hashing it twice is CPU work on a single-threaded runtime, so the walk
+// yields between batches. Without it a large scope holds the event loop for as long as it runs and
+// every other request waits behind it.
+const PAUSE_BETWEEN_BATCHES_MS = 100;
+
+const NO_PROGRESS = { projectsTotal: 0, projectsDone: 0, secretsProcessed: 0 };
+
+export type TBackfillState = {
+ status: JobState;
+ message?: string;
+ projectsTotal: number;
+ projectsDone: number;
+ secretsProcessed: number;
+};
type TSecretValueTrackingQueueFactoryDep = {
queueService: TQueueServiceFactory;
- keyStore: Pick<
- TKeyStoreFactory,
- "getItemPrimary" | "setItemWithExpiry" | "setItemWithExpiryNX" | "deleteItem" | "deleteItems"
- >;
+ keyStore: Pick<TKeyStoreFactory, "deleteItems">;
projectDAL: Pick<TProjectDALFactory, "find" | "findById" | "updateById">;
orgDAL: Pick<TOrgDALFactory, "updateById">;
folderDAL: Pick<TSecretFolderDALFactory, "findByProjectId">;
@@ -40,10 +40,8 @@ type TSecretValueTrackingQueueFactoryDep = {
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
};
-// A payload with no scope is one queued before the org-wide walk shipped. It is read as project
-// scope so the shape is understood, but such a job has no run state and the handler stops on that:
-// the old code kept no state, so there is no cursor to resume and nothing to safely assume. The
-// enable button simply comes back for the customer to press again.
+// A payload with no scope predates the org-wide walk. Reading it as project scope is what lets a job
+// queued seconds before a deploy still do the thing it was queued to do.
const resolveScope = (data: { scope?: "org" | "project"; orgId?: string; projectId?: string }): TBackfillScope => {
if (data.scope === "org" && data.orgId) return { scope: "org", orgId: data.orgId };
return { scope: "project", projectId: data.projectId as string };
@@ -51,6 +49,10 @@ const resolveScope = (data: { scope?: "org" | "project"; orgId?: string; project
const scopeIdOf = (scope: TBackfillScope) => (scope.scope === "org" ? scope.orgId : scope.projectId);
+// One job id per scope. BullMQ will not add a second job under an id it already holds, and that is
+// what stops two backfills running over the same scope.
+const jobIdOf = (scope: TBackfillScope) => `secret-value-tracking-${scopeIdOf(scope)}`;
+
export const secretValueTrackingQueueFactory = ({
queueService,
keyStore,
@@ -60,267 +62,190 @@ export const secretValueTrackingQueueFactory = ({
secretV2BridgeDAL,
kmsService
}: TSecretValueTrackingQueueFactoryDep) => {
- const state = secretValueTrackingStateFactory({ keyStore });
+ const startBackfill = async (scope: TBackfillScope) => {
+ const jobId = jobIdOf(scope);
+
+ const existing = await queueService.getJob(QueueName.SecretBlindIndexMigration, jobId);
+ if (existing) {
+ const state = await existing.getState();
+ // A finished record is kept only so a status poll can still see it. It is not a run, so it
+ // must not stand in the way of starting one.
+ if (state === JobState.Completed || state === JobState.Failed) await existing.remove();
+ else return;
+ }
- // Each chunk needs its own id: BullMQ ignores an add for an id that is currently active, so a
- // fixed id would break the chain at the second chunk.
- const queueChunk = async (scope: TBackfillScope) =>
- queueService.queue(QueueName.SecretBlindIndexMigration, QueueJobs.SecretBlindIndexMigration, scope, {
+ await queueService.queue(QueueName.SecretBlindIndexMigration, QueueJobs.SecretBlindIndexMigration, scope, {
+ // Brief, because completion itself is the durable flag; this only has to outlive a poll.
removeOnComplete: { age: 60 },
+ // A day, so someone who comes back the next morning can still read why it failed.
removeOnFail: { age: 24 * 3600 },
- // Without retries one transient database or KMS blip ends the chain and the customer has to
- // start the backfill again by hand.
attempts: 3,
backoff: { type: "exponential", delay: 5000 },
- jobId: `secret-value-tracking-${scopeIdOf(scope)}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`
+ jobId
});
-
- const $resolveProjectIds = async (scope: TBackfillScope) => {
- if (scope.scope === "project") {
- const project = await projectDAL.findById(scope.projectId);
- return { orgId: project?.orgId ?? null, projectIds: project ? [project.id] : [] };
- }
-
- const projects = await projectDAL.find({ orgId: scope.orgId, type: ProjectType.SecretManager });
- return {
- orgId: scope.orgId,
- projectIds: projects.map((project) => project.id).sort()
- };
};
- const $markProjectComplete = async (projectId: string) => {
- await projectDAL.updateById(projectId, { secretBlindIndexEnabled: true });
- await keyStore.deleteItems({ pattern: `${KeyStorePrefixes.InsightsCache(projectId, "secrets-duplication")}*` });
- };
+ const getBackfillState = async (scope: TBackfillScope): Promise<TBackfillState> => {
+ const job = await queueService.getJob(QueueName.SecretBlindIndexMigration, jobIdOf(scope));
+ if (!job) return { status: JobState.NotFound, ...NO_PROGRESS };
- const handler = async (job: { data: { scope?: "org" | "project"; orgId?: string; projectId?: string } }) => {
- const scope = resolveScope(job.data);
- const scopeId = scopeIdOf(scope);
+ // The job reports its own progress as it walks, so there is nowhere else to keep it.
+ const reported = (job.progress ?? {}) as Partial<typeof NO_PROGRESS>;
+ const progress = { ...NO_PROGRESS, ...reported };
- const runState = await state.read(scopeId);
- if (!runState) {
- logger.info(`SecretValueTrackingBackfill: no run state, nothing to resume [scopeId=${scopeId}]`);
- return;
+ const state = await job.getState();
+ if (state === JobState.Failed) {
+ return { status: JobState.Failed, message: job.failedReason ?? "Unknown error", ...progress };
}
+ if (state === JobState.Completed) return { status: JobState.Completed, ...progress };
+ return { status: JobState.Pending, ...progress };
+ };
- // A chunk whose run has since been recorded failed or completed belongs to a chain that has been
- // superseded. Carrying on would walk a cursor another chain now owns, and a finished run's null
- // cursor would restart the whole scope.
- if (runState.status !== "running") {
- logger.info(
- `SecretValueTrackingBackfill: run is ${runState.status}, dropping superseded chunk [scopeId=${scopeId}]`
- );
- return;
- }
+ queueService.start(
+ QueueName.SecretBlindIndexMigration,
+ async (job) => {
+ const scope = resolveScope(job.data);
+ const scopeId = scopeIdOf(scope);
- const { orgId, projectIds } = await $resolveProjectIds(scope);
- if (!orgId) {
- logger.info(`SecretValueTrackingBackfill: scope no longer exists [scopeId=${scopeId}]`);
- await state.clear(scopeId);
- return;
- }
+ const projects =
+ scope.scope === "project"
+ ? [await projectDAL.findById(scope.projectId)].filter(Boolean)
+ : await projectDAL.find({ orgId: scope.orgId, type: ProjectType.SecretManager });
- // One org data key for the whole chunk, since every project in the org shares it.
- const { generateOrgLevelBlindIndex } = await createOrgSecretBlindIndexer({ orgId, kmsService });
- const projectCiphers = new Map<
- string,
- {
- decryptor: (input: { cipherTextBlob: Buffer }) => Buffer;
- generateBlindIndex: (value: Buffer) => Promise<string>;
+ if (!projects.length) {
+ logger.info(`SecretValueTrackingBackfill: nothing to walk [scopeId=${scopeId}]`);
+ return;
}
- >();
- const foldersByProject: Record<string, string[]> = {};
- const $foldersOf = async (projectId: string) => {
- if (!foldersByProject[projectId]) {
+ const { orgId } = projects[0];
+ const projectIds = projects.map((project) => project.id).sort();
+
+ // Every project's folders up front. The walk is one pass over the whole scope, so deferring
+ // them buys nothing, and a complete map is what lets advanceCursor tell a project that has no
+ // folders from one it has simply not been shown.
+ const foldersByProject: Record<string, string[]> = {};
+ for await (const projectId of projectIds) {
// Soft-deleted environments are included: the delete is reversible, so leaving their rows
- // unindexed and then flagging the org complete breaks the moment one is restored.
+ // unindexed and then flagging the scope complete breaks the moment one is restored.
const folders = await folderDAL.findByProjectId(projectId, undefined, true);
foldersByProject[projectId] = folders.map((folder) => folder.id).sort();
}
- return foldersByProject[projectId];
- };
-
- const $cipherOf = async (projectId: string) => {
- const cached = projectCiphers.get(projectId);
- if (cached) return cached;
-
- const { decryptor, generateSecretBlindIndex } = await kmsService.createCipherPairWithDataKey({
- type: KmsDataKey.SecretManager,
- projectId
- });
- const pair = { decryptor, generateBlindIndex: generateSecretBlindIndex };
- projectCiphers.set(projectId, pair);
- return pair;
- };
-
- let { cursor } = runState;
- let { projectsDone } = runState;
- let { secretsProcessed } = runState;
- let readThisChunk = 0;
- let steps = 0;
- let done = false;
-
- const $writeProgress = async (at: TBackfillCursor | null) => {
- await state.write(scopeId, {
- status: "running",
- cursor: at,
- projectsTotal: runState.projectsTotal,
- projectsDone,
- secretsProcessed,
- lastProgressAt: new Date().toISOString()
- });
- };
-
- // advanceCursor only sees the projects whose folders are loaded, and reads an unloaded one as a
- // project with none, so it would declare the walk finished at the first project boundary. Load
- // the next unseeded project and ask again before accepting the end.
- const $advance = async (
- from: TBackfillCursor | null,
- lastRow: { key: string; id: string } | null,
- folderExhausted: boolean
- ) => {
- for (;;) {
- const next = advanceCursor({
- cursor: from,
- projectIds,
- folderIdsByProject: foldersByProject,
- lastRow,
- folderExhausted
+
+ // One org data key for the whole walk, since every project in the org shares it.
+ const { generateOrgLevelBlindIndex } = await createOrgSecretBlindIndexer({ orgId, kmsService });
+ const projectCiphers = new Map<
+ string,
+ {
+ decryptor: (input: { cipherTextBlob: Buffer }) => Buffer;
+ generateBlindIndex: (value: Buffer) => Promise<string>;
+ }
+ >();
+
+ const $cipherOf = async (projectId: string) => {
+ const cached = projectCiphers.get(projectId);
+ if (cached) return cached;
+
+ const { decryptor, generateSecretBlindIndex } = await kmsService.createCipherPairWithDataKey({
+ type: KmsDataKey.SecretManager,
+ projectId
});
- if (!next.done) return next;
+ const pair = { decryptor, generateBlindIndex: generateSecretBlindIndex };
+ projectCiphers.set(projectId, pair);
+ return pair;
+ };
- const unseeded = projectIds.find((projectId) => !(projectId in foldersByProject));
- if (!unseeded) return next;
- // eslint-disable-next-line no-await-in-loop
- await $foldersOf(unseeded);
- }
- };
+ const $markProjectComplete = async (projectId: string) => {
+ await projectDAL.updateById(projectId, { secretBlindIndexEnabled: true });
+ await keyStore.deleteItems({ pattern: `${KeyStorePrefixes.InsightsCache(projectId, "secrets-duplication")}*` });
+ };
- if (!cursor) {
- const seeded = await $advance(null, null, false);
- if (seeded.done) done = true;
- else cursor = seeded.cursor;
- }
+ const $advance = (
+ from: TBackfillCursor | null,
+ lastRow: { key: string; id: string } | null,
+ folderExhausted: boolean
+ ) => advanceCursor({ cursor: from, projectIds, folderIdsByProject: foldersByProject, lastRow, folderExhausted });
- while (!done && cursor && readThisChunk < CHUNK_SIZE && steps < MAX_STEPS_PER_CHUNK) {
- steps += 1;
- // eslint-disable-next-line no-await-in-loop
- await $foldersOf(cursor.projectId);
+ let secretsProcessed = 0;
+ let projectsDone = 0;
- // eslint-disable-next-line no-await-in-loop
- const rows = await secretV2BridgeDAL.findSecretsInFolderAfter(
- cursor.folderId,
- { key: cursor.key, id: cursor.id },
- READ_BATCH_SIZE
- );
- readThisChunk += rows.length;
+ const seeded = $advance(null, null, false);
+ let cursor = seeded.done ? null : seeded.cursor;
- const pending = rows.filter(needsBackfill);
- if (pending.length) {
+ while (cursor) {
// eslint-disable-next-line no-await-in-loop
- const { decryptor, generateBlindIndex } = await $cipherOf(cursor.projectId);
- // eslint-disable-next-line no-await-in-loop
- const updates = await Promise.all(
- pending.map(async (row) => {
- const value = decryptor({ cipherTextBlob: row.encryptedValue as Buffer });
- const [secretValueBlindIndex, secretValueOrgBlindIndex] = await Promise.all([
- generateBlindIndex(value),
- generateOrgLevelBlindIndex(value)
- ]);
- return { id: row.id, secretValueBlindIndex, secretValueOrgBlindIndex };
- })
+ const rows = await secretV2BridgeDAL.findSecretsInFolderAfter(
+ cursor.folderId,
+ { key: cursor.key, id: cursor.id },
+ READ_BATCH_SIZE
);
- // eslint-disable-next-line no-await-in-loop
- await secretV2BridgeDAL.batchSetBlindIndexes(updates);
- secretsProcessed += updates.length;
- }
- const lastRow = rows.length ? { key: rows[rows.length - 1].key, id: rows[rows.length - 1].id } : null;
- // eslint-disable-next-line no-await-in-loop
- const next = await $advance(cursor, lastRow, rows.length < READ_BATCH_SIZE);
+ const pending = rows.filter(needsBackfill);
+ if (pending.length) {
+ // eslint-disable-next-line no-await-in-loop
+ const { decryptor, generateBlindIndex } = await $cipherOf(cursor.projectId);
+ // eslint-disable-next-line no-await-in-loop
+ const updates = await Promise.all(
+ pending.map(async (row) => {
+ const value = decryptor({ cipherTextBlob: row.encryptedValue as Buffer });
+ const [secretValueBlindIndex, secretValueOrgBlindIndex] = await Promise.all([
+ generateBlindIndex(value),
+ generateOrgLevelBlindIndex(value)
+ ]);
+ return { id: row.id, secretValueBlindIndex, secretValueOrgBlindIndex };
+ })
+ );
+ // eslint-disable-next-line no-await-in-loop
+ await secretV2BridgeDAL.batchSetBlindIndexes(updates);
+ secretsProcessed += updates.length;
+ }
+
+ const lastRow = rows.length ? { key: rows[rows.length - 1].key, id: rows[rows.length - 1].id } : null;
+ const next = $advance(cursor, lastRow, rows.length < READ_BATCH_SIZE);
- if (next.done) {
- // The walk ends inside the project it was working, so that project's own flag is set here
- // rather than from a completedProjectId the `done` branch never carries. A cursor left on a
- // project that has since been deleted is not a project to flag.
- if (projectIds.includes(cursor.projectId)) {
+ if (next.done) {
+ // The walk ends inside the project it was working, so that project's own flag is set here
+ // rather than from a completedProjectId the `done` branch never carries.
// eslint-disable-next-line no-await-in-loop
await $markProjectComplete(cursor.projectId);
projectsDone += 1;
+ cursor = null;
+ break;
}
- done = true;
- break;
- }
- if (next.completedProjectId) {
- // eslint-disable-next-line no-await-in-loop
- await $markProjectComplete(next.completedProjectId);
- projectsDone += 1;
- }
- cursor = next.cursor;
+ if (next.completedProjectId) {
+ // eslint-disable-next-line no-await-in-loop
+ await $markProjectComplete(next.completedProjectId);
+ projectsDone += 1;
+ }
+ cursor = next.cursor;
- if (steps % HEARTBEAT_EVERY_STEPS === 0) {
+ // Doubles as the heartbeat BullMQ reads to tell a working job from a stalled one.
+ // eslint-disable-next-line no-await-in-loop
+ await job.updateProgress({ projectsTotal: projectIds.length, projectsDone, secretsProcessed });
// eslint-disable-next-line no-await-in-loop
- await $writeProgress(cursor);
+ await new Promise((resolve) => {
+ setTimeout(resolve, PAUSE_BETWEEN_BATCHES_MS);
+ });
}
- }
- if (done) {
if (scope.scope === "org") await orgDAL.updateById(scope.orgId, { orgWideSecretValueTrackingEnabled: true });
- // Kept rather than cleared, so the UI can show what the run got through. The flag above is
- // what actually answers "is it done", and this key is free to expire.
- await state.write(scopeId, {
- status: "completed",
- cursor: null,
- projectsTotal: runState.projectsTotal,
- projectsDone,
- secretsProcessed,
- lastProgressAt: new Date().toISOString()
- });
+
+ await job.updateProgress({ projectsTotal: projectIds.length, projectsDone, secretsProcessed });
logger.info(
`SecretValueTrackingBackfill: complete [scopeId=${scopeId}] [projectsDone=${projectsDone}] [secretsProcessed=${secretsProcessed}]`
);
- return;
- }
-
- await $writeProgress(cursor);
- await queueChunk(scope);
- };
-
- queueService.start(QueueName.SecretBlindIndexMigration, handler, {
- concurrency: 1,
- limiter: { max: 1, duration: 1000 }
- });
+ },
+ // One walk at a time, so a second scope waits rather than competing for the same connection pool
+ // and the same CPU.
+ { concurrency: 1 }
+ );
queueService.listen(QueueName.SecretBlindIndexMigration, "failed", (job, err) => {
- const scopeId = job?.data ? scopeIdOf(resolveScope(job.data)) : undefined;
- logger.error(err, `SecretValueTrackingBackfill: chunk failed [scopeId=${scopeId}]`);
- if (!scopeId) return;
-
- // BullMQ raises this on every failed attempt, not only the last. Recording a failure while a
- // retry is still queued would let the enable guard hand the scope to a second chain, and the two
- // would then share one cursor.
- const attemptsAllowed = job?.opts?.attempts ?? 1;
- if ((job?.attemptsMade ?? 0) < attemptsAllowed) return;
-
- // Recording the failure is what lets the status endpoint answer immediately, rather than making
- // the user wait out the staleness window.
- void state
- .read(scopeId)
- .then((existing) =>
- existing ? state.write(scopeId, { ...existing, status: "failed", error: err.message }) : undefined
- )
- .catch((writeErr) =>
- logger.error(writeErr, `SecretValueTrackingBackfill: could not record failure [scopeId=${scopeId}]`)
- );
+ const scopeId = job?.data ? scopeIdOf(resolveScope(job.data)) : "unknown";
+ logger.error(err, `SecretValueTrackingBackfill: failed [scopeId=${scopeId}]`);
});
- const startBackfill = async (scope: TBackfillScope) => {
- await queueChunk(scope);
- };
-
- return { startBackfill };
+ return { startBackfill, getBackfillState };
};
export type TSecretValueTrackingQueueFactory = ReturnType<typeof secretValueTrackingQueueFactory>;
diff --git a/backend/src/services/secret-value-tracking/secret-value-tracking-service.ts b/backend/src/services/secret-value-tracking/secret-value-tracking-service.ts
index 5c2877bf9a..a78c5fae2d 100644
--- a/backend/src/services/secret-value-tracking/secret-value-tracking-service.ts
+++ b/backend/src/services/secret-value-tracking/secret-value-tracking-service.ts
@@ -4,21 +4,18 @@ import { ActionProjectType, OrganizationActionScope, ProjectType } from "@app/db
import { OrgPermissionActions, OrgPermissionSubjects } from "@app/ee/services/permission/org-permission";
import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service-types";
import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission";
-import { TKeyStoreFactory } from "@app/keystore/keystore";
-import { BadRequestError, NotFoundError } from "@app/lib/errors";
+import { NotFoundError } from "@app/lib/errors";
import { OrgServiceActor, TProjectPermission } from "@app/lib/types";
+import { JobState } from "@app/queue/queue-service";
import { TOrgDALFactory } from "../org/org-dal";
import { TProjectDALFactory } from "../project/project-dal";
-import { resolveRunStatus } from "./secret-value-tracking-fns";
import { TSecretValueTrackingQueueFactory } from "./secret-value-tracking-queue";
-import { secretValueTrackingStateFactory } from "./secret-value-tracking-state";
type TSecretValueTrackingServiceFactoryDep = {
permissionService: Pick<TPermissionServiceFactory, "getOrgPermission" | "getProjectPermission">;
orgDAL: Pick<TOrgDALFactory, "findById">;
projectDAL: Pick<TProjectDALFactory, "find" | "findById">;
- keyStore: Pick<TKeyStoreFactory, "getItemPrimary" | "setItemWithExpiry" | "setItemWithExpiryNX" | "deleteItem">;
secretValueTrackingQueue: TSecretValueTrackingQueueFactory;
};
@@ -26,17 +23,8 @@ export const secretValueTrackingServiceFactory = ({
permissionService,
orgDAL,
projectDAL,
- keyStore,
secretValueTrackingQueue
}: TSecretValueTrackingServiceFactoryDep) => {
- const state = secretValueTrackingStateFactory({ keyStore });
-
- const $progressOf = (runState: Awaited<ReturnType<typeof state.read>>) => ({
- projectsTotal: runState?.projectsTotal ?? 0,
- projectsDone: runState?.projectsDone ?? 0,
- secretsProcessed: runState?.secretsProcessed ?? 0
- });
-
const enableForOrg = async (actor: OrgServiceActor) => {
const { permission } = await permissionService.getOrgPermission({
actor: actor.type,
@@ -55,30 +43,14 @@ export const secretValueTrackingServiceFactory = ({
// unindexed row back into a completed org (a rollback to a version predating the digest, an
// environment restored after the walk passed it), and without a re-run the only repair is SQL.
// The walk skips rows that already carry both digests, so a redundant run costs a read pass.
-
- // Projects are counted so progress can be reported as "project M of N". Secrets deliberately are
- // not: counting them is expensive on exactly the organizations where progress matters, and the
- // number is stale the moment it is taken.
+ //
+ // A run that is already moving is refused by the queue instead: one job id per scope, and BullMQ
+ // will not add a second job under an id it already holds.
const projects = await projectDAL.find({ orgId: actor.orgId, type: ProjectType.SecretManager });
- const projectsTotal = projects.length;
-
- const claimed = await state.claim(actor.orgId, projectsTotal);
- if (!claimed) {
- throw new BadRequestError({
- message: "A secret value tracking backfill is already running for this organization"
- });
- }
-
- // Releasing the claim matters: without it a failed enqueue leaves the organization holding a
- // claim it is not using, and the guard refuses every retry until the staleness window passes.
- try {
- await secretValueTrackingQueue.startBackfill({ scope: "org", orgId: actor.orgId });
- } catch (error) {
- await state.clear(actor.orgId);
- throw error;
- }
-
- return { projectsTotal };
+
+ await secretValueTrackingQueue.startBackfill({ scope: "org", orgId: actor.orgId });
+
+ return { projectsTotal: projects.length };
};
const getOrgStatus = async (actor: OrgServiceActor) => {
@@ -95,11 +67,10 @@ export const secretValueTrackingServiceFactory = ({
const org = await orgDAL.findById(actor.orgId);
if (!org) throw new NotFoundError({ message: `Organization with ID '${actor.orgId}' not found` });
- const runState = await state.read(actor.orgId);
- return {
- ...resolveRunStatus(runState, Boolean(org.orgWideSecretValueTrackingEnabled), new Date()),
- ...$progressOf(runState)
- };
+ // The flag is the durable answer. The queue only says whether a run is in flight, or why the
+ // last one stopped, and it drops a finished job within the minute.
+ const progress = await secretValueTrackingQueue.getBackfillState({ scope: "org", orgId: actor.orgId });
+ return org.orgWideSecretValueTrackingEnabled ? { ...progress, status: JobState.Completed } : progress;
};
const $assertProjectSettingsEdit = async ({
@@ -127,28 +98,14 @@ export const secretValueTrackingServiceFactory = ({
const enableForProject = async (dto: TProjectPermission) => {
const project = await $assertProjectSettingsEdit(dto);
-
- const claimed = await state.claim(project.id, 1);
- if (!claimed) {
- throw new BadRequestError({ message: "A secret value tracking backfill is already running for this project" });
- }
-
- try {
- await secretValueTrackingQueue.startBackfill({ scope: "project", projectId: project.id });
- } catch (error) {
- await state.clear(project.id);
- throw error;
- }
+ await secretValueTrackingQueue.startBackfill({ scope: "project", projectId: project.id });
};
const getProjectStatus = async (dto: TProjectPermission) => {
const project = await $assertProjectSettingsEdit(dto);
- const runState = await state.read(project.id);
- return {
- ...resolveRunStatus(runState, Boolean(project.secretBlindIndexEnabled), new Date()),
- ...$progressOf(runState)
- };
+ const progress = await secretValueTrackingQueue.getBackfillState({ scope: "project", projectId: project.id });
+ return project.secretBlindIndexEnabled ? { ...progress, status: JobState.Completed } : progress;
};
return { enableForOrg, getOrgStatus, enableForProject, getProjectStatus };
diff --git a/backend/src/services/secret-value-tracking/secret-value-tracking-state.ts b/backend/src/services/secret-value-tracking/secret-value-tracking-state.ts
deleted file mode 100644
index a58ea34e18..0000000000
--- a/backend/src/services/secret-value-tracking/secret-value-tracking-state.ts
+++ /dev/null
@@ -1,87 +0,0 @@
-import { KeyStorePrefixes, KeyStoreTtls, TKeyStoreFactory } from "@app/keystore/keystore";
-import { logger } from "@app/lib/logger";
-
-import { BACKFILL_STALE_AFTER_MS, isUsableRunState } from "./secret-value-tracking-fns";
-import { TBackfillRunState } from "./secret-value-tracking-types";
-
-type TSecretValueTrackingStateFactoryDep = {
- keyStore: Pick<TKeyStoreFactory, "getItemPrimary" | "setItemWithExpiry" | "setItemWithExpiryNX" | "deleteItem">;
-};
-
-export const secretValueTrackingStateFactory = ({ keyStore }: TSecretValueTrackingStateFactoryDep) => {
- // Reads the primary rather than a replica: a chunk writes its cursor immediately before queueing
- // its successor, so a replica read can hand the next chunk the cursor it already worked through.
- const read = async (scopeId: string): Promise<TBackfillRunState | null> => {
- const raw = await keyStore.getItemPrimary(KeyStorePrefixes.SecretValueTrackingBackfill(scopeId));
- if (!raw) return null;
-
- try {
- // An unreadable key means no run in progress, which the status endpoint can answer, rather
- // than an exception it cannot.
- const parsed: unknown = JSON.parse(raw);
- return isUsableRunState(parsed) ? parsed : null;
- } catch (error) {
- logger.warn(error, `Secret value tracking backfill state unreadable [scopeId=${scopeId}]`);
- return null;
- }
- };
-
- const write = async (scopeId: string, state: TBackfillRunState) => {
- await keyStore.setItemWithExpiry(
- KeyStorePrefixes.SecretValueTrackingBackfill(scopeId),
- KeyStoreTtls.SecretValueTrackingBackfillInSeconds,
- JSON.stringify(state)
- );
- };
-
- const clear = async (scopeId: string) => {
- await keyStore.deleteItem(KeyStorePrefixes.SecretValueTrackingBackfill(scopeId));
- };
-
- const isStalled = (state: TBackfillRunState) => {
- // A finished run holds the key only to keep its counters readable, so it never blocks a new one.
- if (state.status === "failed" || state.status === "completed") return true;
- const lastProgress = new Date(state.lastProgressAt).getTime();
- return Number.isNaN(lastProgress) || Date.now() - lastProgress > BACKFILL_STALE_AFTER_MS;
- };
-
- // Taking over a stalled run keeps its cursor, so retrying a backfill that died hours ago resumes
- // instead of walking the whole scope again.
- const claim = async (scopeId: string, projectsTotal: number) => {
- const fresh: TBackfillRunState = {
- status: "running",
- cursor: null,
- projectsTotal,
- projectsDone: 0,
- secretsProcessed: 0,
- lastProgressAt: new Date().toISOString()
- };
-
- const claimed = await keyStore.setItemWithExpiryNX(
- KeyStorePrefixes.SecretValueTrackingBackfill(scopeId),
- KeyStoreTtls.SecretValueTrackingBackfillInSeconds,
- JSON.stringify(fresh)
- );
- if (claimed) return true;
-
- const existing = await read(scopeId);
- if (!existing) {
- await write(scopeId, fresh);
- return true;
- }
-
- if (!isStalled(existing)) return false;
-
- await write(scopeId, {
- ...fresh,
- cursor: existing.cursor,
- projectsDone: existing.projectsDone,
- secretsProcessed: existing.secretsProcessed
- });
- return true;
- };
-
- return { read, write, clear, claim };
-};
-
-export type TSecretValueTrackingStateFactory = ReturnType<typeof secretValueTrackingStateFactory>;
diff --git a/backend/src/services/secret-value-tracking/secret-value-tracking-types.ts b/backend/src/services/secret-value-tracking/secret-value-tracking-types.ts
index 622303deff..495955e39a 100644
--- a/backend/src/services/secret-value-tracking/secret-value-tracking-types.ts
+++ b/backend/src/services/secret-value-tracking/secret-value-tracking-types.ts
@@ -7,18 +7,6 @@ export type TBackfillCursor = {
export type TBackfillScope = { scope: "org"; orgId: string } | { scope: "project"; projectId: string };
-export type TBackfillRunState = {
- // "completed" carries the final counters so the UI can show what the run got through. It is never
- // the answer to "is it done": that is the durable flag, and this key is allowed to expire.
- status: "running" | "failed" | "completed";
- cursor: TBackfillCursor | null;
- projectsTotal: number;
- projectsDone: number;
- secretsProcessed: number;
- lastProgressAt: string;
- error?: string;
-};
-
export type TAdvanceCursorInput = {
cursor: TBackfillCursor | null;
projectIds: string[];