Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.
TypeScript acmeagentsai-agentscertificate-managementdevsecopsenvironment-variableskubernetesopen-sourcepampkipostgresprivate-caprivileged-access-managementsecret-managementsecret-managersecretssecrets-managementsecuritysecurity-toolsvault
fix(secret-value-tracking): address review from akhilmhdh
Calvin826daeb7a6b5
diff --git a/.gitignore b/.gitignoreindex 990231dd63..b90ecb360e 100644--- a/.gitignore+++ b/.gitignore@@ -82,9 +82,3 @@ __pycache__# Local Codex design QA artifacts/.design-qa//design-qa.md--# superpowers plan workspaces (ledger, briefs, review packages)-.superpowers/--# trellis workspace-.trellis/diff --git a/backend/e2e-test/insights-secrets-duplication.spec.ts b/backend/e2e-test/insights-secrets-duplication.spec.tsindex e832873fc4..70e7853575 100644--- a/backend/e2e-test/insights-secrets-duplication.spec.ts+++ b/backend/e2e-test/insights-secrets-duplication.spec.ts@@ -189,7 +189,8 @@ describe("insights secrets duplication", () => {describe("organization scope", () => {test("groups a value across projects and ranks the widest spread first", async () => {- const { result } = await insightsService.getOrgSecretsDuplication({ ...orgActor, refresh: true });+ await clearCaches();+ const { result } = await insightsService.getOrgSecretsDuplication(orgActor);expect(result.orgWideSecretValueTrackingEnabled).toBe(true);expect(result.groups.map((g) => [g.projectCount, g.locationCount, g.secrets.length])).toEqual([@@ -206,25 +207,20 @@ describe("insights secrets duplication", () => {);});- test("reports when the answer was computed", async () => {- const before = Date.now();- const { result } = await insightsService.getOrgSecretsDuplication({ ...orgActor, refresh: true });-- expect(result.computedAt).toEqual(expect.any(String));- expect(new Date(result.computedAt as string).getTime()).toBeGreaterThanOrEqual(before - 1000);- });-- test("answers from the cache until asked to refresh", async () => {- await insightsService.getOrgSecretsDuplication({ ...orgActor, refresh: true });+ test("answers from the cache until it expires", async () => {+ await clearCaches();+ await insightsService.getOrgSecretsDuplication(orgActor);const extra = secret(projects.elsewhere.folderId, "LATE_ARRIVAL", "shared");await testDb(TableName.SecretV2).insert(extra);try {const cached = await insightsService.getOrgSecretsDuplication(orgActor);expect(cached.result.groups[0].secrets).toHaveLength(2);+ expect(cached.remainingTTL).toBeGreaterThan(0);- const refreshed = await insightsService.getOrgSecretsDuplication({ ...orgActor, refresh: true });- expect(refreshed.result.groups[0].secrets).toHaveLength(3);+ await clearCaches();+ const recomputed = await insightsService.getOrgSecretsDuplication(orgActor);+ expect(recomputed.result.groups[0].secrets).toHaveLength(3);} finally {await testDb(TableName.SecretV2).where("id", extra.id).delete();}@@ -234,9 +230,7 @@ describe("insights secrets duplication", () => {test("refuses a viewer who may read insights but not search all secret values", async () => {grantedInsightsActions = [OrgPermissionSecretsManagementInsightsActions.Read];try {- await expect(insightsService.getOrgSecretsDuplication({ ...orgActor, refresh: true })).rejects.toThrow(- ForbiddenError- );+ await expect(insightsService.getOrgSecretsDuplication(orgActor)).rejects.toThrow(ForbiddenError);} finally {grantedInsightsActions = [OrgPermissionSecretsManagementInsightsActions.SearchAllSecretValues];}@@ -246,7 +240,7 @@ describe("insights secrets duplication", () => {await testDb(TableName.Organization).where("id", ORG_ID).update({ orgWideSecretValueTrackingEnabled: false });try {const { result } = await insightsService.getOrgSecretsDuplication(orgActor);- expect(result).toEqual({ orgWideSecretValueTrackingEnabled: false, groups: [], computedAt: null });+ expect(result).toEqual({ orgWideSecretValueTrackingEnabled: false, groups: [] });} finally {await testDb(TableName.Organization).where("id", ORG_ID).update({ orgWideSecretValueTrackingEnabled: true });}@@ -256,16 +250,9 @@ describe("insights secrets duplication", () => {describe("project scope", () => {const dto = { projectId: projects.here.projectId };- test("reports when the answer was computed", async () => {- const before = Date.now();- const { result } = await insightsService.getSecretsDuplication({ ...dto, refresh: true }, projectActor);-- expect(result.computedAt).toEqual(expect.any(String));- expect(new Date(result.computedAt as string).getTime()).toBeGreaterThanOrEqual(before - 1000);- });-- test("answers from the cache until asked to refresh", async () => {- await insightsService.getSecretsDuplication({ ...dto, refresh: true }, projectActor);+ test("answers from the cache until it expires", async () => {+ await clearCaches();+ await insightsService.getSecretsDuplication(dto, projectActor);const extra = secret(projects.here.folderId, "LATE_LOCAL", "local");await testDb(TableName.SecretV2).insert(extra);@@ -274,9 +261,9 @@ describe("insights secrets duplication", () => {r.result.groups.find((g) => g.secrets.some((s) => s.key === "LOCAL_A"))?.secrets.length;expect(localGroupSize(await insightsService.getSecretsDuplication(dto, projectActor))).toBe(3);- expect(- localGroupSize(await insightsService.getSecretsDuplication({ ...dto, refresh: true }, projectActor))- ).toBe(4);++ await clearCaches();+ expect(localGroupSize(await insightsService.getSecretsDuplication(dto, projectActor))).toBe(4);} finally {await testDb(TableName.SecretV2).where("id", extra.id).delete();}diff --git a/backend/e2e-test/testUtils/insights.ts b/backend/e2e-test/testUtils/insights.tsindex 7fa5b90cd0..5cdf42c389 100644--- a/backend/e2e-test/testUtils/insights.ts+++ b/backend/e2e-test/testUtils/insights.ts@@ -146,6 +146,5 @@ export const buildOrgInsightsGateStubs = () => {export const passThroughKeyStore: TInsightsServiceFactoryDep["keyStore"] = {getItem: async () => null,setItemWithExpiry: async () => "OK",- ttl: async () => -2,- deleteItem: async () => 1+ ttl: async () => -2};diff --git a/backend/src/ee/routes/v1/insights-router.ts b/backend/src/ee/routes/v1/insights-router.tsindex 50197ae195..84f550df59 100644--- a/backend/src/ee/routes/v1/insights-router.ts+++ b/backend/src/ee/routes/v1/insights-router.ts@@ -508,21 +508,9 @@ export const registerInsightsRouter = async (server: FastifyZodProvider) => {operationId: "getOrgInsightsSecretsDuplication",description: "Get groups of secrets that share a value anywhere in the organization",security: [{ bearerAuth: [] }],- querystring: z.object({- refresh: z- .enum(["true", "false"])- .default("false")- .transform((value) => value === "true")- .describe("Recompute now instead of answering from the cached result.")- }),response: {200: z.object({orgWideSecretValueTrackingEnabled: z.boolean(),- computedAt: z- .string()- .datetime()- .nullable()- .describe("When the groups were computed. Null when detection is not enabled."),groups: z.array(z.object({projectCount: z.number(),@@ -548,8 +536,7 @@ export const registerInsightsRouter = async (server: FastifyZodProvider) => {actor: req.permission.type,actorId: req.permission.id,actorAuthMethod: req.permission.authMethod,- actorOrgId: req.permission.orgId,- refresh: req.query.refresh+ actorOrgId: req.permission.orgId});await server.services.auditLog.createAuditLog({@@ -580,21 +567,9 @@ export const registerInsightsRouter = async (server: FastifyZodProvider) => {params: z.object({projectId: z.string().trim()}),- querystring: z.object({- refresh: z- .enum(["true", "false"])- .default("false")- .transform((value) => value === "true")- .describe("Recompute now instead of answering from the cached result.")- }),response: {200: z.object({secretBlindIndexEnabled: z.boolean(),- computedAt: z- .string()- .datetime()- .nullable()- .describe("When the groups were computed. Null when detection is not enabled."),groups: z.array(z.object({secrets: z.array(@@ -616,7 +591,7 @@ export const registerInsightsRouter = async (server: FastifyZodProvider) => {handler: async (req, reply) => {const { projectId } = req.params;const { result, remainingTTL } = await server.services.insights.getSecretsDuplication(- { projectId, refresh: req.query.refresh },+ { projectId },req.permission);await server.services.auditLog.createAuditLog({diff --git a/backend/src/ee/services/insights/insights-service.test.ts b/backend/src/ee/services/insights/insights-service.test.tsindex c171627bdf..e6877a0344 100644--- a/backend/src/ee/services/insights/insights-service.test.ts+++ b/backend/src/ee/services/insights/insights-service.test.ts@@ -120,8 +120,7 @@ const buildService = (catalog: TSecretsProjectWarning[]) => {keyStore: {getItem: async () => null,setItemWithExpiry: async () => "OK",- ttl: async () => -2,- deleteItem: async () => 1+ ttl: async () => -2},orgDAL: {countSecretManagerProjectMembers: unreachable("orgDAL.countSecretManagerProjectMembers"),diff --git a/backend/src/ee/services/insights/insights-service.ts b/backend/src/ee/services/insights/insights-service.tsindex aaea998ac3..60d576f72e 100644--- a/backend/src/ee/services/insights/insights-service.ts+++ b/backend/src/ee/services/insights/insights-service.ts@@ -92,7 +92,7 @@ export type TInsightsServiceFactoryDep = {projectDAL: Pick<TProjectDALFactory, "findById">;userDAL: Pick<TUserDALFactory, "find">;kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;- keyStore: Pick<TKeyStoreFactory, "setItemWithExpiry" | "getItem" | "ttl" | "deleteItem">;+ keyStore: Pick<TKeyStoreFactory, "setItemWithExpiry" | "getItem" | "ttl">;orgDAL: Pick<TOrgDALFactory, "countSecretManagerProjectMembers" | "findById">;identityOrgMembershipDAL: Pick<TIdentityOrgDALFactory, "countSecretManagerProjectIdentities">;dynamicSecretLeaseDAL: Pick<TDynamicSecretLeaseDALFactory, "countLeasesForOrg">;@@ -497,20 +497,17 @@ export const insightsServiceFactory = ({// Org-scoped digests only exist once the backfill has run, so an empty answer before then would// read as "no duplicates" when it means "nothing has been indexed".if (!org.orgWideSecretValueTrackingEnabled) {- return { result: { orgWideSecretValueTrackingEnabled: false as const, groups: [], computedAt: null } };+ return { result: { orgWideSecretValueTrackingEnabled: false as const, groups: [] } };}const cacheKey = KeyStorePrefixes.InsightsCache(dto.orgId, "org-secrets-duplication");- if (dto.refresh) await keyStore.deleteItem(cacheKey);-const result = await withCache({keyStore,key: cacheKey,ttlSeconds: KeyStoreTtls.InsightsDuplicationCacheInSeconds,fetcher: async () => {- const computedAt = new Date().toISOString();const rawGroups = await secretV2BridgeDAL.findDuplicatedSecretValuesInOrg(dto.orgId);- if (!rawGroups.length) return { orgWideSecretValueTrackingEnabled: true as const, groups: [], computedAt };+ if (!rawGroups.length) return { orgWideSecretValueTrackingEnabled: true as const, groups: [] };// One decryptor per project rather than per group: the value is encrypted under the owning// project's data key, and resolving that key can reach an external KMS.@@ -579,14 +576,13 @@ export const insightsServiceFactory = ({// window, which is a different problem from three copies inside one project..sort((a, b) => b.projectCount - a.projectCount || b.locationCount - a.locationCount);- return { orgWideSecretValueTrackingEnabled: true as const, groups, computedAt };+ return { orgWideSecretValueTrackingEnabled: true as const, groups };}});const remainingTTL = await getCacheTtl(keyStore, cacheKey);- // An entry cached before computedAt existed lacks it until it expires.- return { result: { ...result, computedAt: result.computedAt ?? null }, remainingTTL };+ return { result, remainingTTL };};const getSecretsDuplication = async (dto: TGetSecretsDuplicationDTO, actorDto: OrgServiceActor) => {@@ -600,20 +596,16 @@ export const insightsServiceFactory = ({return {result: {secretBlindIndexEnabled: false,- groups: [],- computedAt: null+ groups: []}};}- if (dto.refresh) await keyStore.deleteItem(cacheKey);-const result = await withCache({keyStore,key: cacheKey,ttlSeconds: KeyStoreTtls.InsightsDuplicationCacheInSeconds,fetcher: async () => {- const computedAt = new Date().toISOString();const rawGroups = await secretV2BridgeDAL.findDuplicatedSecretValues(dto.projectId);const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({@@ -649,14 +641,13 @@ export const insightsServiceFactory = ({}))}));- return { secretBlindIndexEnabled: true as const, groups, computedAt };+ return { secretBlindIndexEnabled: true as const, groups };}});const remainingTTL = await getCacheTtl(keyStore, cacheKey);- // An entry cached before computedAt existed lacks it until it expires.- return { result: { ...result, computedAt: result.computedAt ?? null }, remainingTTL };+ return { result, remainingTTL };};const getCounts = async (dto: TGetInsightsCountsDTO, actorDto: OrgServiceActor) => {diff --git a/backend/src/ee/services/insights/insights-types.ts b/backend/src/ee/services/insights/insights-types.tsindex 4ec162e9e3..025f9efdd1 100644--- a/backend/src/ee/services/insights/insights-types.ts+++ b/backend/src/ee/services/insights/insights-types.ts@@ -36,7 +36,6 @@ export type TGetInsightsSummaryDTO = {export type TGetSecretsDuplicationDTO = {projectId: string;- refresh?: boolean;};export type TGetInsightsCountsDTO = {@@ -47,11 +46,7 @@ export type TGetInsightsCountsDTO = {export type TOrgInsightsDTO = TOrgPermission;-// `refresh` lets the card recompute on demand rather than wait out the cache, which is the whole-// point of the control that replaced the countdown badge.-export type TGetOrgSecretsDuplicationDTO = TOrgInsightsDTO & {- refresh?: boolean;-};+export type TGetOrgSecretsDuplicationDTO = TOrgInsightsDTO;export type TGetSecretsProjectWarningsDTO = TOrgInsightsDTO & {offset: number;diff --git a/backend/src/keystore/keystore.ts b/backend/src/keystore/keystore.tsindex f1fbfba369..18d72bfaf4 100644--- a/backend/src/keystore/keystore.ts+++ b/backend/src/keystore/keystore.ts@@ -254,10 +254,7 @@ export const KeyStoreTtls = {EmailDispatchMailboxWindowInSeconds: 86400, // 24 hoursEmailDispatchAbuseProbeInSeconds: 7200, // 2 hoursInsightsCacheInSeconds: 300, // 5 minutes- // Five minutes, matching every other insight on the page. An hour was far more than the query- // costs, and nothing invalidates this key when a secret changes, so a longer window meant the card- // could sit on a stale empty answer long after the duplicates it exists to find were created.- InsightsDuplicationCacheInSeconds: 300,+ InsightsDuplicationCacheInSeconds: 300, // 5 minutesInsightsWeeklyHistoryCacheInSeconds: 86400, // 24 hoursInsightsOrgCacheInSeconds: 900, // 15 minutesAdminConfigInSeconds: 60,diff --git a/backend/src/services/folder-commit/folder-commit-service.ts b/backend/src/services/folder-commit/folder-commit-service.tsindex 71d335ac32..f4cb199607 100644--- a/backend/src/services/folder-commit/folder-commit-service.ts+++ b/backend/src/services/folder-commit/folder-commit-service.ts@@ -1014,12 +1014,8 @@ export const folderCommitServiceFactory = ({}};- /**- * Process secret changes when applying folder state differences- */- // A version row written before the org-scoped digest existed carries none, and a rollback copies a- // version's digests onto the live secret. Left alone that puts an unindexed row back into a- // project the org has already been told is fully searchable, and nothing would ever fix it.+ // A rollback recomputes the digests from the restored value instead of copying them from the version+ // row, so the secret version table (which is very large) never needed a digest migration.const $buildBlindIndexRepairer = async (projectId: string, tx?: Knex): Promise<TBlindIndexRepairer> => {const project = await projectDAL.findById(projectId, tx);if (!project) return (version) => version;@@ -1031,13 +1027,15 @@ export const folderCommitServiceFactory = ({return async (version) => {if (!version.encryptedValue) return version;- if (version.secretValueBlindIndex && version.secretValueOrgBlindIndex) return version;const value = decryptor({ cipherTextBlob: version.encryptedValue });return { ...version, ...(await blindIndexer.generateBlindIndexes(value)) };};};+ /**+ * Process secret changes when applying folder state differences+ */const processSecretChanges = async (changes: ResourceChange[],secretVersions: Record<string, TSecretVersionsV2>,diff --git a/backend/src/services/secret-value-tracking/secret-value-tracking-fns.test.ts b/backend/src/services/secret-value-tracking/secret-value-tracking-fns.test.tsindex b20c0b4654..bfb73853bb 100644--- a/backend/src/services/secret-value-tracking/secret-value-tracking-fns.test.ts+++ b/backend/src/services/secret-value-tracking/secret-value-tracking-fns.test.ts@@ -1,142 +1,4 @@-import { advanceCursor, needsBackfill } from "./secret-value-tracking-fns";--const PROJECTS = ["p1", "p2"];-const FOLDERS = { p1: ["f1", "f2"], p2: ["f3"] };--describe("advanceCursor", () => {- test("a null cursor starts at the first folder of the first project", () => {- expect(- advanceCursor({- cursor: null,- projectIds: PROJECTS,- folderIdsByProject: FOLDERS,- lastRow: null,- folderExhausted: false- })- ).toEqual({ done: false, cursor: { projectId: "p1", folderId: "f1", key: "", id: "" }, completedProjectId: null });- });-- test("a partly read folder advances within itself", () => {- expect(- advanceCursor({- cursor: { projectId: "p1", folderId: "f1", key: "A", id: "s1" },- projectIds: PROJECTS,- folderIdsByProject: FOLDERS,- lastRow: { key: "B", id: "s2" },- folderExhausted: false- })- ).toEqual({- done: false,- cursor: { projectId: "p1", folderId: "f1", key: "B", id: "s2" },- completedProjectId: null- });- });-- test("an exhausted folder moves to the next folder of the same project", () => {- expect(- advanceCursor({- cursor: { projectId: "p1", folderId: "f1", key: "Z", id: "s9" },- projectIds: PROJECTS,- folderIdsByProject: FOLDERS,- lastRow: null,- folderExhausted: true- })- ).toEqual({ done: false, cursor: { projectId: "p1", folderId: "f2", key: "", id: "" }, completedProjectId: null });- });-- test("the last folder of a project reports the project complete and moves to the next project", () => {- expect(- advanceCursor({- cursor: { projectId: "p1", folderId: "f2", key: "Z", id: "s9" },- projectIds: PROJECTS,- folderIdsByProject: FOLDERS,- lastRow: null,- folderExhausted: true- })- ).toEqual({ done: false, cursor: { projectId: "p2", folderId: "f3", key: "", id: "" }, completedProjectId: "p1" });- });-- test("the last folder of the last project ends the walk", () => {- expect(- advanceCursor({- cursor: { projectId: "p2", folderId: "f3", key: "Z", id: "s9" },- projectIds: PROJECTS,- folderIdsByProject: FOLDERS,- lastRow: null,- folderExhausted: true- })- ).toEqual({ done: true });- });-- // An empty folder must not leave the cursor where it was, or the walk spins on it forever.- test("an empty folder advances rather than repeating itself", () => {- expect(- advanceCursor({- cursor: { projectId: "p1", folderId: "f1", key: "", id: "" },- projectIds: PROJECTS,- folderIdsByProject: FOLDERS,- lastRow: null,- folderExhausted: true- })- ).toEqual({ done: false, cursor: { projectId: "p1", folderId: "f2", key: "", id: "" }, completedProjectId: null });- });-- // Folder ids are sorted, so resuming past the deleted one keeps the walk moving forward instead of- // re-reading everything the run already covered in that project.- test("a cursor on a deleted folder resumes at the next folder after it, not at the first", () => {- expect(- advanceCursor({- cursor: { projectId: "p1", folderId: "f1a", key: "M", id: "s5" },- projectIds: PROJECTS,- folderIdsByProject: FOLDERS,- lastRow: null,- folderExhausted: false- })- ).toEqual({ done: false, cursor: { projectId: "p1", folderId: "f2", key: "", id: "" }, completedProjectId: null });- });-- test("a cursor on a deleted folder past the project's last one moves to the next project", () => {- expect(- advanceCursor({- cursor: { projectId: "p1", folderId: "zz-gone", key: "M", id: "s5" },- projectIds: PROJECTS,- folderIdsByProject: FOLDERS,- lastRow: null,- folderExhausted: false- })- ).toEqual({ done: false, cursor: { projectId: "p2", folderId: "f3", key: "", id: "" }, completedProjectId: "p1" });- });-- test("a cursor on a deleted project resumes at the next project", () => {- expect(- advanceCursor({- cursor: { projectId: "gone", folderId: "gone", key: "M", id: "s5" },- projectIds: PROJECTS,- folderIdsByProject: FOLDERS,- lastRow: null,- folderExhausted: false- })- ).toEqual({ done: false, cursor: { projectId: "p1", folderId: "f1", key: "", id: "" }, completedProjectId: null });- });-- test("a project with no folders is skipped", () => {- expect(- advanceCursor({- cursor: { projectId: "p1", folderId: "f2", key: "Z", id: "s9" },- projectIds: ["p1", "empty", "p2"],- folderIdsByProject: { ...FOLDERS, empty: [] },- lastRow: null,- folderExhausted: true- })- ).toEqual({ done: false, cursor: { projectId: "p2", folderId: "f3", key: "", id: "" }, completedProjectId: "p1" });- });-- test("an organization with no projects is immediately done", () => {- expect(- advanceCursor({ cursor: null, projectIds: [], folderIdsByProject: {}, lastRow: null, folderExhausted: false })- ).toEqual({ done: true });- });-});+import { needsBackfill } from "./secret-value-tracking-fns";describe("needsBackfill", () => {test("a row missing the org digest needs work", () => {diff --git a/backend/src/services/secret-value-tracking/secret-value-tracking-fns.ts b/backend/src/services/secret-value-tracking/secret-value-tracking-fns.tsindex 018ce5f86a..ebd2fdfd12 100644--- a/backend/src/services/secret-value-tracking/secret-value-tracking-fns.ts+++ b/backend/src/services/secret-value-tracking/secret-value-tracking-fns.ts@@ -1,88 +1,3 @@-import { TAdvanceCursorInput, TAdvanceCursorResult, TBackfillCursor } from "./secret-value-tracking-types";--const startOfFolder = (projectId: string, folderId: string): TBackfillCursor => ({- projectId,- folderId,- key: "",- id: ""-});--// Resolves to the first project from `fromIndex` that still has folders, so a project deleted or-// emptied between chunks is stepped over rather than stalling the walk on something that is no-// longer there.-const firstProjectWithFolders = (- projectIds: string[],- folderIdsByProject: Record<string, string[]>,- fromIndex: number-) => {- for (let i = fromIndex; i < projectIds.length; i += 1) {- const projectId = projectIds[i];- const folderIds = folderIdsByProject[projectId] ?? [];- if (folderIds.length) return { projectId, folderId: folderIds[0] };- }- return null;-};--export const advanceCursor = ({- cursor,- projectIds,- folderIdsByProject,- lastRow,- folderExhausted-}: TAdvanceCursorInput): TAdvanceCursorResult => {- if (!cursor) {- const next = firstProjectWithFolders(projectIds, folderIdsByProject, 0);- if (!next) return { done: true };- return { done: false, cursor: startOfFolder(next.projectId, next.folderId), completedProjectId: null };- }-- const projectIndex = projectIds.indexOf(cursor.projectId);- if (projectIndex === -1) {- const next = firstProjectWithFolders(projectIds, folderIdsByProject, 0);- if (!next) return { done: true };- return { done: false, cursor: startOfFolder(next.projectId, next.folderId), completedProjectId: null };- }-- const folderIds = folderIdsByProject[cursor.projectId] ?? [];- const folderIndex = folderIds.indexOf(cursor.folderId);-- // The cursor's folder is gone. Folder ids are sorted, so resuming at the first one past it keeps- // the walk moving forward rather than re-reading everything this project already covered.- if (folderIndex === -1) {- const nextFolderId = folderIds.find((folderId) => folderId > cursor.folderId);- if (nextFolderId) {- return { done: false, cursor: startOfFolder(cursor.projectId, nextFolderId), completedProjectId: null };- }- const next = firstProjectWithFolders(projectIds, folderIdsByProject, projectIndex + 1);- if (!next) return { done: true };- return {- done: false,- cursor: startOfFolder(next.projectId, next.folderId),- completedProjectId: folderIds.length ? cursor.projectId : null- };- }-- if (!folderExhausted && lastRow) {- return {- done: false,- cursor: { projectId: cursor.projectId, folderId: cursor.folderId, key: lastRow.key, id: lastRow.id },- completedProjectId: null- };- }-- if (folderIndex + 1 < folderIds.length) {- return {- done: false,- cursor: startOfFolder(cursor.projectId, folderIds[folderIndex + 1]),- completedProjectId: null- };- }-- const next = firstProjectWithFolders(projectIds, folderIdsByProject, projectIndex + 1);- if (!next) return { done: true };- return { done: false, cursor: startOfFolder(next.projectId, next.folderId), completedProjectId: cursor.projectId };-};-export const needsBackfill = (row: {secretValueBlindIndex?: string | null;secretValueOrgBlindIndex?: string | null;diff --git a/backend/src/services/secret-value-tracking/secret-value-tracking-queue.test.ts b/backend/src/services/secret-value-tracking/secret-value-tracking-queue.test.tsindex 2422d4dd49..7a97c9b1cf 100644--- a/backend/src/services/secret-value-tracking/secret-value-tracking-queue.test.ts+++ b/backend/src/services/secret-value-tracking/secret-value-tracking-queue.test.ts@@ -22,6 +22,7 @@ const makeHarness = ({const flaggedProjects: string[] = [];const flaggedOrgs: string[] = [];const folderReads: string[] = [];+ const events: string[] = [];const progress: { projectsTotal: number; projectsDone: number; secretsProcessed: number }[] = [];let rows = [...secrets];let handler: (job: {@@ -50,6 +51,7 @@ const makeHarness = ({findById: async (id: string) => ({ id, orgId: ORG_ID }) as never,updateById: async (id: string) => {flaggedProjects.push(id);+ events.push(`flagged:${id}`);return {} as never;}} as never,@@ -62,6 +64,7 @@ const makeHarness = ({folderDAL: {findByProjectId: async (projectId: string) => {folderReads.push(projectId);+ events.push(`folders:${projectId}`);return (foldersByProject[projectId] ?? []).map((id) => ({ id })) as never;}} as never,@@ -98,6 +101,7 @@ const makeHarness = ({flaggedProjects,flaggedOrgs,folderReads,+ events,progress,rowsNow: () => rows,run: () =>@@ -194,6 +198,21 @@ describe("the backfill job", () => {expect(harness.progress.at(-1)?.secretsProcessed).toBe(0);});+ test("each project is finished before the next one's folders are loaded", async () => {+ const harness = makeHarness({+ projectIds: ["p1", "p2"],+ foldersByProject: { p1: ["f1"], p2: ["f2"] },+ secrets: [+ { id: "s1", key: "A", folderId: "f1", hasOrgDigest: false },+ { id: "s2", key: "B", folderId: "f2", hasOrgDigest: false }+ ]+ });++ await harness.run();++ expect(harness.events).toEqual(["folders:p1", "flagged:p1", "folders:p2", "flagged:p2"]);+ });+test("progress is reported as the walk goes, not only at the end", async () => {const harness = makeHarness({projectIds: ["p1", "p2"],diff --git a/backend/src/services/secret-value-tracking/secret-value-tracking-queue.ts b/backend/src/services/secret-value-tracking/secret-value-tracking-queue.tsindex b387af9dbf..ad890b41b9 100644--- a/backend/src/services/secret-value-tracking/secret-value-tracking-queue.ts+++ b/backend/src/services/secret-value-tracking/secret-value-tracking-queue.ts@@ -11,8 +11,8 @@ import { TProjectDALFactory } from "../project/project-dal";import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";import { createOrgSecretBlindIndexer } from "../secret-v2-bridge/secret-blind-index-fns";import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal";-import { advanceCursor, needsBackfill } from "./secret-value-tracking-fns";-import { TBackfillCursor, TBackfillScope } from "./secret-value-tracking-types";+import { needsBackfill } from "./secret-value-tracking-fns";+import { TBackfillScope } from "./secret-value-tracking-types";const READ_BATCH_SIZE = 1000;// Decrypting a batch and hashing it twice is CPU work on a single-threaded runtime, so the walk@@ -22,6 +22,11 @@ const PAUSE_BETWEEN_BATCHES_MS = 100;const NO_PROGRESS = { projectsTotal: 0, projectsDone: 0, secretsProcessed: 0 };+type TProjectCipher = Pick<+ Awaited<ReturnType<TKmsServiceFactory["createCipherPairWithDataKey"]>>,+ "decryptor" | "generateSecretBlindIndex"+>;+export type TBackfillState = {status: JobState;message?: string;@@ -119,138 +124,82 @@ export const secretValueTrackingQueueFactory = ({return;}- const { orgId } = projects[0];- const projectIds = projects.map((project) => project.id).sort();-- // Every project's folders up front. The walk is one pass over the whole scope, so deferring- // them buys nothing, and a complete map is what lets advanceCursor tell a project that has no- // folders from one it has simply not been shown.- const foldersByProject: Record<string, string[]> = {};- for await (const projectId of projectIds) {- // Soft-deleted environments are included: the delete is reversible, so leaving their rows- // unindexed and then flagging the scope complete breaks the moment one is restored.- const folders = await folderDAL.findByProjectId(projectId, undefined, true);- foldersByProject[projectId] = folders.map((folder) => folder.id).sort();- }-// One org data key for the whole walk, since every project in the org shares it.- const { generateOrgLevelBlindIndex } = await createOrgSecretBlindIndexer({ orgId, kmsService });- const projectCiphers = new Map<- string,- {- decryptor: (input: { cipherTextBlob: Buffer }) => Buffer;- generateBlindIndex: (value: Buffer) => Promise<string>;- }- >();-- const $cipherOf = async (projectId: string) => {- const cached = projectCiphers.get(projectId);- if (cached) return cached;-- const { decryptor, generateSecretBlindIndex } = await kmsService.createCipherPairWithDataKey({- type: KmsDataKey.SecretManager,- projectId- });- const pair = { decryptor, generateBlindIndex: generateSecretBlindIndex };- projectCiphers.set(projectId, pair);- return pair;- };-- const markedProjectIds = new Set<string>();- const $markProjectComplete = async (projectId: string) => {- markedProjectIds.add(projectId);- await projectDAL.updateById(projectId, { secretBlindIndexEnabled: true });- await keyStore.deleteItems({ pattern: `${KeyStorePrefixes.InsightsCache(projectId, "secrets-duplication")}*` });- };+ const { generateOrgLevelBlindIndex } = await createOrgSecretBlindIndexer({+ orgId: projects[0].orgId,+ kmsService+ });+ const progress = { projectsTotal: projects.length, projectsDone: 0, secretsProcessed: 0 };- const $advance = (- from: TBackfillCursor | null,- lastRow: { key: string; id: string } | null,- folderExhausted: boolean- ) => advanceCursor({ cursor: from, projectIds, folderIdsByProject: foldersByProject, lastRow, folderExhausted });+ const $backfillFolder = async (folderId: string, { decryptor, generateSecretBlindIndex }: TProjectCipher) => {+ let after = { key: "", id: "" };+ let hasMore = true;- let secretsProcessed = 0;- let projectsDone = 0;-- const seeded = $advance(null, null, false);- let cursor = seeded.done ? null : seeded.cursor;-- while (cursor) {- // eslint-disable-next-line no-await-in-loop- const rows = await secretV2BridgeDAL.findSecretsInFolderAfter(- cursor.folderId,- { key: cursor.key, id: cursor.id },- READ_BATCH_SIZE- );-- const pending = rows.filter(needsBackfill);- if (pending.length) {+ while (hasMore) {// eslint-disable-next-line no-await-in-loop- const { decryptor, generateBlindIndex } = await $cipherOf(cursor.projectId);+ const rows = await secretV2BridgeDAL.findSecretsInFolderAfter(folderId, after, READ_BATCH_SIZE);++ const pending = rows.filter(needsBackfill);+ if (pending.length) {+ // eslint-disable-next-line no-await-in-loop+ const updates = await Promise.all(+ pending.map(async (row) => {+ const value = decryptor({ cipherTextBlob: row.encryptedValue as Buffer });+ const [secretValueBlindIndex, secretValueOrgBlindIndex] = await Promise.all([+ generateSecretBlindIndex(value),+ generateOrgLevelBlindIndex(value)+ ]);+ return {+ id: row.id,+ encryptedValue: row.encryptedValue as Buffer,+ secretValueBlindIndex,+ secretValueOrgBlindIndex+ };+ })+ );+ // eslint-disable-next-line no-await-in-loop+ await secretV2BridgeDAL.batchSetBlindIndexes(updates);+ progress.secretsProcessed += updates.length;+ }++ hasMore = rows.length === READ_BATCH_SIZE;+ if (hasMore) after = { key: rows[rows.length - 1].key, id: rows[rows.length - 1].id };++ // Doubles as the heartbeat BullMQ reads to tell a working job from a stalled one.// eslint-disable-next-line no-await-in-loop- const updates = await Promise.all(- pending.map(async (row) => {- const value = decryptor({ cipherTextBlob: row.encryptedValue as Buffer });- const [secretValueBlindIndex, secretValueOrgBlindIndex] = await Promise.all([- generateBlindIndex(value),- generateOrgLevelBlindIndex(value)- ]);- return {- id: row.id,- encryptedValue: row.encryptedValue as Buffer,- secretValueBlindIndex,- secretValueOrgBlindIndex- };- })- );+ await job.updateProgress({ ...progress });// eslint-disable-next-line no-await-in-loop- await secretV2BridgeDAL.batchSetBlindIndexes(updates);- secretsProcessed += updates.length;+ await new Promise((resolve) => {+ setTimeout(resolve, PAUSE_BETWEEN_BATCHES_MS);+ });}+ };- const lastRow = rows.length ? { key: rows[rows.length - 1].key, id: rows[rows.length - 1].id } : null;- const next = $advance(cursor, lastRow, rows.length < READ_BATCH_SIZE);+ for await (const project of projects) {+ // Soft-deleted environments are included: the delete is reversible, so leaving their rows+ // unindexed and then flagging the scope complete breaks the moment one is restored.+ const folders = await folderDAL.findByProjectId(project.id, undefined, true);+ const cipher = await kmsService.createCipherPairWithDataKey({+ type: KmsDataKey.SecretManager,+ projectId: project.id+ });- if (next.done) {- // The walk ends inside the project it was working, so that project's own flag is set here- // rather than from a completedProjectId the `done` branch never carries.- // eslint-disable-next-line no-await-in-loop- await $markProjectComplete(cursor.projectId);- projectsDone += 1;- cursor = null;- break;+ for await (const folder of folders) {+ await $backfillFolder(folder.id, cipher);}- if (next.completedProjectId) {- // eslint-disable-next-line no-await-in-loop- await $markProjectComplete(next.completedProjectId);- projectsDone += 1;- }- cursor = next.cursor;-- // Doubles as the heartbeat BullMQ reads to tell a working job from a stalled one.- // eslint-disable-next-line no-await-in-loop- await job.updateProgress({ projectsTotal: projectIds.length, projectsDone, secretsProcessed });- // eslint-disable-next-line no-await-in-loop- await new Promise((resolve) => {- setTimeout(resolve, PAUSE_BETWEEN_BATCHES_MS);+ await projectDAL.updateById(project.id, { secretBlindIndexEnabled: true });+ await keyStore.deleteItems({+ pattern: `${KeyStorePrefixes.InsightsCache(project.id, "secrets-duplication")}*`});- }-- // The cursor steps over projects with no folders, so they are never finished inside the walk.- // They hold nothing to index, and every secret written to them later carries both digests.- for await (const projectId of projectIds) {- if (!markedProjectIds.has(projectId)) {- await $markProjectComplete(projectId);- projectsDone += 1;- }+ progress.projectsDone += 1;+ await job.updateProgress({ ...progress });}if (scope.scope === "org") await orgDAL.updateById(scope.orgId, { orgWideSecretValueTrackingEnabled: true });- await job.updateProgress({ projectsTotal: projectIds.length, projectsDone, secretsProcessed });logger.info(- `SecretValueTrackingBackfill: complete [scopeId=${scopeId}] [projectsDone=${projectsDone}] [secretsProcessed=${secretsProcessed}]`+ `SecretValueTrackingBackfill: complete [scopeId=${scopeId}] [projectsDone=${progress.projectsDone}] [secretsProcessed=${progress.secretsProcessed}]`);},// One walk at a time, so a second scope waits rather than competing for the same connection pooldiff --git a/backend/src/services/secret-value-tracking/secret-value-tracking-service.ts b/backend/src/services/secret-value-tracking/secret-value-tracking-service.tsindex d7c8f565f4..1c3dda5f46 100644--- a/backend/src/services/secret-value-tracking/secret-value-tracking-service.ts+++ b/backend/src/services/secret-value-tracking/secret-value-tracking-service.ts@@ -7,7 +7,7 @@ import {} from "@app/ee/services/permission/org-permission";import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service-types";import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission";-import { NotFoundError } from "@app/lib/errors";+import { BadRequestError, NotFoundError } from "@app/lib/errors";import { OrgServiceActor, TProjectPermission } from "@app/lib/types";import { JobState } from "@app/queue/queue-service";@@ -47,15 +47,14 @@ export const secretValueTrackingServiceFactory = ({const org = await orgDAL.findById(actor.orgId);if (!org) throw new NotFoundError({ message: `Organization with ID '${actor.orgId}' not found` });- // Deliberately not refused when the flag is already set. Several ordinary operations can put an- // unindexed row back into a completed org (a rollback to a version predating the digest, an- // environment restored after the walk passed it), and without a re-run the only repair is SQL.- // The walk skips rows that already carry both digests, so a redundant run costs a read pass.- //- // A run that is already moving is refused by the queue instead: one job id per scope, and BullMQ- // will not add a second job under an id it already holds.+ if (org.orgWideSecretValueTrackingEnabled) {+ throw new BadRequestError({ message: "Secret value search is already enabled for this organization" });+ }+const projects = await projectDAL.find({ orgId: actor.orgId, type: ProjectType.SecretManager });+ // A run that is already moving is refused by the queue: one job id per scope, and BullMQ will not+ // add a second job under an id it already holds.await secretValueTrackingQueue.startBackfill({ scope: "org", orgId: actor.orgId });return { projectsTotal: projects.length };@@ -109,6 +108,10 @@ export const secretValueTrackingServiceFactory = ({const enableForProject = async (dto: TProjectPermission) => {const project = await $assertProjectSettingsEdit(dto);+ if (project.secretBlindIndexEnabled) {+ throw new BadRequestError({ message: "Secret blind indexing is already enabled for this project" });+ }+await secretValueTrackingQueue.startBackfill({ scope: "project", projectId: project.id });};diff --git a/backend/src/services/secret-value-tracking/secret-value-tracking-types.ts b/backend/src/services/secret-value-tracking/secret-value-tracking-types.tsindex 495955e39a..9312ce8356 100644--- a/backend/src/services/secret-value-tracking/secret-value-tracking-types.ts+++ b/backend/src/services/secret-value-tracking/secret-value-tracking-types.ts@@ -1,22 +1 @@-export type TBackfillCursor = {- projectId: string;- folderId: string;- key: string;- id: string;-};-export type TBackfillScope = { scope: "org"; orgId: string } | { scope: "project"; projectId: string };--export type TAdvanceCursorInput = {- cursor: TBackfillCursor | null;- projectIds: string[];- folderIdsByProject: Record<string, string[]>;- lastRow: { key: string; id: string } | null;- folderExhausted: boolean;-};--// `completedProjectId` is how the caller learns the walk just left a project behind, which is the-// moment that project's own blind index flag can be set.-export type TAdvanceCursorResult =- | { done: true }- | { done: false; cursor: TBackfillCursor; completedProjectId: string | null };diff --git a/docs/superpowers/plans/2026-09-24-org-wide-secret-value-tracking-backfill.md b/docs/superpowers/plans/2026-09-24-org-wide-secret-value-tracking-backfill.mddeleted file mode 100644index a4ba9d2564..0000000000--- a/docs/superpowers/plans/2026-09-24-org-wide-secret-value-tracking-backfill.md+++ /dev/null@@ -1,1180 +0,0 @@-# Org-wide Secret Value Tracking Backfill Implementation Plan--> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.--**Goal:** Repurpose the project blind index backfill into one chunked, resumable job that walks either a single project or a whole organization, and gate secret value search on the organization's backfill having completed.--**Architecture:** One BullMQ job walks projects, then folders, then rows, using keyset pagination over the existing `idx_secrets_v2_folder_id_key`. Each job processes up to 5000 secrets, writes its cursor to a Redis key, and queues its successor, so no single job holds a worker for long. Completion is a durable boolean on `organizations`; everything transient (status, cursor, progress, last progress time) lives in the Redis key with a 2 day TTL.--**Tech Stack:** TypeScript, Fastify 4, Knex + PostgreSQL, BullMQ, ioredis via the keystore, Vitest, Zod.--**Spec:** `docs/superpowers/specs/2026-09-24-org-wide-secret-value-tracking-backfill-design.md`--## Global Constraints--- Path alias `@app/*` maps to `backend/src/*`. Imports are sorted by `simple-import-sort` in the group order documented in `backend/CLAUDE.md`.-- Every database call inside a `transaction()` callback passes `tx`. No network calls, sleeps, or unbounded row counts between BEGIN and COMMIT (`backend/CODE_QUALITY.md`).-- Every route needs `config.rateLimit`, `onRequest: verifyAuth([...])`, an `operationId`, and `.describe()` on body and querystring fields.-- Every user-facing error names the thing that is wrong in product vocabulary. No raw Knex errors reach the client.-- Default to no comments. A comment earns its place only by explaining why.-- Never use em dashes in any prose, comment, or message.-- The organization column is named for the capability, not the mechanism: `orgWideSecretValueTrackingEnabled`. Nothing on the organization schema says "blind index".-- Chunk size is 5000 secrets per job, read in database batches of 1000.-- Redis key TTL is 2 days (`172800` seconds). The staleness window for the double-trigger guard is 15 minutes (`900` seconds).-- `make reviewable-api` must pass before the final commit.--## Review Focus--These are the input classes the spec implies but that no task's happy-path test exercises. Each one has its test added to the task that owns the code.--1. A cursor naming a project or folder that has been deleted since the last chunk. The walk must step past it and continue, not throw. Covered in Task 2.-2. A folder that is empty, or whose every row already carries both digests. The walk must advance to the next folder rather than spin on the same cursor forever. Covered in Task 2.-3. A personal secret sharing a key with a shared secret in the same folder. The `(key, id)` cursor must visit both; a key-only cursor would skip one. This needs a real database, so it is covered by the e2e spec in Task 8 rather than by Task 4, which has no test of its own.-4. `lastProgressAt` exactly at the staleness boundary. The guard must be decisive rather than flapping between running and stalled. Covered in Task 3.-5. A malformed or truncated Redis payload (an older shape, a partial write). Reading it must degrade to "no run in progress" rather than throwing into the status endpoint. Covered in Task 3.-------## File Structure--**Create:**-- `backend/src/db/migrations/20260924120000_add-org-wide-secret-value-tracking.ts`- The organization column.-- `backend/src/services/secret-value-tracking/secret-value-tracking-types.ts`- Cursor, run state, scope, and status types.-- `backend/src/services/secret-value-tracking/secret-value-tracking-fns.ts`- Pure walk and status logic.-- `backend/src/services/secret-value-tracking/secret-value-tracking-fns.test.ts`- Unit tests for the above.-- `backend/src/services/secret-value-tracking/secret-value-tracking-state.ts`- The Redis key: claim, read, advance, clear.-- `backend/src/services/secret-value-tracking/secret-value-tracking-queue.ts`- The chunked job.-- `backend/src/services/secret-value-tracking/secret-value-tracking-service.ts`- Enable and status for both scopes, permissions, audit.-- `backend/e2e-test/routes/v1/org-secret-value-tracking.spec.ts`- End to end.--**Modify:**-- `backend/src/db/schemas/organizations.ts`- Regenerated, not hand edited.-- `backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts`- Keyset query, relaxed update guard.-- `backend/src/services/project/project-queue.ts`- The blind index job and its helpers move out.-- `backend/src/services/project/project-service.ts`- Delegate to the new service.-- `backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts`- The search gate.-- `backend/src/server/routes/v1/organization-router.ts`- The two new routes.-- `backend/src/server/routes/index.ts`- Wiring.-- `backend/src/keystore/keystore.ts`- The key prefix.-- `backend/src/queue/queue-service.ts`- The widened payload.-- `backend/src/ee/services/audit-log/audit-log-types.ts`- The audit event.-------### Task 1: The organization flag--**Files:**-- Create: `backend/src/db/migrations/20260924120000_add-org-wide-secret-value-tracking.ts`-- Modify: `backend/src/db/schemas/organizations.ts` (regenerated)--**Interfaces:**-- Consumes: nothing.-- Produces: `organizations.orgWideSecretValueTrackingEnabled: boolean`, reachable as `OrganizationsSchema.shape.orgWideSecretValueTrackingEnabled` and on `TOrganizations`.--- [ ] **Step 1: Write the migration**--Copy the shape of `backend/src/db/migrations/20260604100000_add-secret-blind-index-enabled-to-project.ts`. New rows default to true because an organization created after this ships is complete by construction: every secret it writes carries both digests. Existing rows are set to false because their older secrets have no organization digest.--```ts-import { Knex } from "knex";--import { TableName } from "../schemas";--export async function up(knex: Knex): Promise<void> {- if (await knex.schema.hasTable(TableName.Organization)) {- const hasColumn = await knex.schema.hasColumn(TableName.Organization, "orgWideSecretValueTrackingEnabled");- if (!hasColumn) {- await knex.schema.alterTable(TableName.Organization, (t) => {- t.boolean("orgWideSecretValueTrackingEnabled").defaultTo(true).notNullable();- });-- // Organizations that already exist hold secrets written before the org-scoped digest, so they- // are incomplete until their backfill runs. Only organizations created from here on are- // complete by construction.- await knex(TableName.Organization).update({ orgWideSecretValueTrackingEnabled: false });- }- }-}--export async function down(knex: Knex): Promise<void> {- if (await knex.schema.hasTable(TableName.Organization)) {- const hasColumn = await knex.schema.hasColumn(TableName.Organization, "orgWideSecretValueTrackingEnabled");- if (hasColumn) {- await knex.schema.alterTable(TableName.Organization, (t) => {- t.dropColumn("orgWideSecretValueTrackingEnabled");- });- }- }-}-```--- [ ] **Step 2: Regenerate the schema**--Run: `cd backend && npm run migration:latest-dev && npm run generate:schema`--Expected: `backend/src/db/schemas/organizations.ts` gains `orgWideSecretValueTrackingEnabled: z.boolean().default(true),`. Do not hand edit that file.--- [ ] **Step 3: Verify the migration runs from empty**--Run: `make test-api-e2e SPEC=status`--Expected: PASS. The e2e environment drops the schema and runs every migration before the suite, so a green run is proof the migration applies cleanly from nothing.--- [ ] **Step 4: Commit**--```bash-git add backend/src/db/migrations/20260924120000_add-org-wide-secret-value-tracking.ts backend/src/db/schemas/organizations.ts-git commit -m "feat(secret-value-tracking): add the org-wide tracking flag"-```-------### Task 2: Pure walk logic--**Files:**-- Create: `backend/src/services/secret-value-tracking/secret-value-tracking-types.ts`-- Create: `backend/src/services/secret-value-tracking/secret-value-tracking-fns.ts`-- Test: `backend/src/services/secret-value-tracking/secret-value-tracking-fns.test.ts`--**Interfaces:**-- Consumes: nothing.-- Produces:- - `type TBackfillCursor = { projectId: string; folderId: string; key: string; id: string }`- - `type TBackfillScope = { scope: "org"; orgId: string } | { scope: "project"; projectId: string }`- - `type TBackfillRunState = { status: "running" | "failed"; cursor: TBackfillCursor | null; projectsTotal: number; projectsDone: number; secretsProcessed: number; lastProgressAt: string; error?: string }`- - `advanceCursor(input: TAdvanceCursorInput): TAdvanceCursorResult`- - `needsBackfill(row: { secretValueBlindIndex?: string | null; secretValueOrgBlindIndex?: string | null; encryptedValue?: Buffer | null }): boolean`--- [ ] **Step 1: Write the types**--```ts-export type TBackfillCursor = {- projectId: string;- folderId: string;- key: string;- id: string;-};--export type TBackfillScope = { scope: "org"; orgId: string } | { scope: "project"; projectId: string };--export type TBackfillRunState = {- status: "running" | "failed";- cursor: TBackfillCursor | null;- projectsTotal: number;- projectsDone: number;- secretsProcessed: number;- lastProgressAt: string;- error?: string;-};--export type TAdvanceCursorInput = {- cursor: TBackfillCursor | null;- projectIds: string[];- folderIdsByProject: Record<string, string[]>;- lastRow: { key: string; id: string } | null;- folderExhausted: boolean;-};--export type TAdvanceCursorResult =- | { done: true }- | { done: false; cursor: TBackfillCursor; completedProjectId: string | null };-```--`completedProjectId` is how the caller learns it just finished a project's last folder, which is the moment to flip that project's `secretBlindIndexEnabled`.--- [ ] **Step 2: Write the failing tests**--```ts-import { advanceCursor, needsBackfill } from "./secret-value-tracking-fns";--const PROJECTS = ["p1", "p2"];-const FOLDERS = { p1: ["f1", "f2"], p2: ["f3"] };--describe("advanceCursor", () => {- test("a null cursor starts at the first folder of the first project", () => {- expect(- advanceCursor({- cursor: null,- projectIds: PROJECTS,- folderIdsByProject: FOLDERS,- lastRow: null,- folderExhausted: false- })- ).toEqual({ done: false, cursor: { projectId: "p1", folderId: "f1", key: "", id: "" }, completedProjectId: null });- });-- test("a partly read folder advances within itself", () => {- expect(- advanceCursor({- cursor: { projectId: "p1", folderId: "f1", key: "A", id: "s1" },- projectIds: PROJECTS,- folderIdsByProject: FOLDERS,- lastRow: { key: "B", id: "s2" },- folderExhausted: false- })- ).toEqual({ done: false, cursor: { projectId: "p1", folderId: "f1", key: "B", id: "s2" }, completedProjectId: null });- });-- test("an exhausted folder moves to the next folder of the same project", () => {- expect(- advanceCursor({- cursor: { projectId: "p1", folderId: "f1", key: "Z", id: "s9" },- projectIds: PROJECTS,- folderIdsByProject: FOLDERS,- lastRow: null,- folderExhausted: true- })- ).toEqual({ done: false, cursor: { projectId: "p1", folderId: "f2", key: "", id: "" }, completedProjectId: null });- });-- test("the last folder of a project reports the project complete and moves to the next project", () => {- expect(- advanceCursor({- cursor: { projectId: "p1", folderId: "f2", key: "Z", id: "s9" },- projectIds: PROJECTS,- folderIdsByProject: FOLDERS,- lastRow: null,- folderExhausted: true- })- ).toEqual({ done: false, cursor: { projectId: "p2", folderId: "f3", key: "", id: "" }, completedProjectId: "p1" });- });-- test("the last folder of the last project ends the walk", () => {- expect(- advanceCursor({- cursor: { projectId: "p2", folderId: "f3", key: "Z", id: "s9" },- projectIds: PROJECTS,- folderIdsByProject: FOLDERS,- lastRow: null,- folderExhausted: true- })- ).toEqual({ done: true });- });-- // Review Focus 2: an empty folder must not leave the cursor where it was, or the walk spins.- test("an empty folder advances rather than repeating itself", () => {- const result = advanceCursor({- cursor: { projectId: "p1", folderId: "f1", key: "", id: "" },- projectIds: PROJECTS,- folderIdsByProject: FOLDERS,- lastRow: null,- folderExhausted: true- });- expect(result).toEqual({- done: false,- cursor: { projectId: "p1", folderId: "f2", key: "", id: "" },- completedProjectId: null- });- });-- // Review Focus 1: a project or folder deleted between chunks.- test("a cursor on a deleted folder resumes at the next folder of that project", () => {- expect(- advanceCursor({- cursor: { projectId: "p1", folderId: "gone", key: "M", id: "s5" },- projectIds: PROJECTS,- folderIdsByProject: FOLDERS,- lastRow: null,- folderExhausted: false- })- ).toEqual({ done: false, cursor: { projectId: "p1", folderId: "f1", key: "", id: "" }, completedProjectId: null });- });-- test("a cursor on a deleted project resumes at the next project", () => {- expect(- advanceCursor({- cursor: { projectId: "gone", folderId: "gone", key: "M", id: "s5" },- projectIds: PROJECTS,- folderIdsByProject: FOLDERS,- lastRow: null,- folderExhausted: false- })- ).toEqual({ done: false, cursor: { projectId: "p1", folderId: "f1", key: "", id: "" }, completedProjectId: null });- });-- test("a project with no folders is skipped", () => {- expect(- advanceCursor({- cursor: { projectId: "p1", folderId: "f2", key: "Z", id: "s9" },- projectIds: ["p1", "empty", "p2"],- folderIdsByProject: { ...FOLDERS, empty: [] },- lastRow: null,- folderExhausted: true- })- ).toEqual({ done: false, cursor: { projectId: "p2", folderId: "f3", key: "", id: "" }, completedProjectId: "p1" });- });-- test("an organization with no projects is immediately done", () => {- expect(- advanceCursor({ cursor: null, projectIds: [], folderIdsByProject: {}, lastRow: null, folderExhausted: false })- ).toEqual({ done: true });- });-});--describe("needsBackfill", () => {- test("a row missing the org digest needs work", () => {- expect(- needsBackfill({ secretValueBlindIndex: "abc", secretValueOrgBlindIndex: null, encryptedValue: Buffer.from("x") })- ).toBe(true);- });-- test("a row missing the project digest needs work", () => {- expect(- needsBackfill({ secretValueBlindIndex: null, secretValueOrgBlindIndex: "abc", encryptedValue: Buffer.from("x") })- ).toBe(true);- });-- test("a row with both digests is skipped", () => {- expect(- needsBackfill({ secretValueBlindIndex: "a", secretValueOrgBlindIndex: "b", encryptedValue: Buffer.from("x") })- ).toBe(false);- });-- test("a row with no encrypted value is skipped even when digests are missing", () => {- expect(needsBackfill({ secretValueBlindIndex: null, secretValueOrgBlindIndex: null, encryptedValue: null })).toBe(- false- );- });-});-```--- [ ] **Step 3: Run the tests to verify they fail**--Run: `make test-api-unit SPEC=secret-value-tracking-fns`--Expected: FAIL, because `./secret-value-tracking-fns` does not exist.--- [ ] **Step 4: Write the implementation**--```ts-import { TAdvanceCursorInput, TAdvanceCursorResult, TBackfillCursor } from "./secret-value-tracking-types";--const startOfFolder = (projectId: string, folderId: string): TBackfillCursor => ({- projectId,- folderId,- key: "",- id: ""-});--// Resolves to the first project at or after the cursor's project, so a project deleted between-// chunks is stepped over rather than stalling the walk on a project that is no longer there.-const firstProjectWithFolders = (- projectIds: string[],- folderIdsByProject: Record<string, string[]>,- fromIndex: number-) => {- for (let i = fromIndex; i < projectIds.length; i += 1) {- const projectId = projectIds[i];- const folderIds = folderIdsByProject[projectId] ?? [];- if (folderIds.length) return { projectId, folderId: folderIds[0] };- }- return null;-};--export const advanceCursor = ({- cursor,- projectIds,- folderIdsByProject,- lastRow,- folderExhausted-}: TAdvanceCursorInput): TAdvanceCursorResult => {- if (!cursor) {- const next = firstProjectWithFolders(projectIds, folderIdsByProject, 0);- if (!next) return { done: true };- return { done: false, cursor: startOfFolder(next.projectId, next.folderId), completedProjectId: null };- }-- const projectIndex = projectIds.indexOf(cursor.projectId);- if (projectIndex === -1) {- const next = firstProjectWithFolders(projectIds, folderIdsByProject, 0);- if (!next) return { done: true };- return { done: false, cursor: startOfFolder(next.projectId, next.folderId), completedProjectId: null };- }-- const folderIds = folderIdsByProject[cursor.projectId] ?? [];- const folderIndex = folderIds.indexOf(cursor.folderId);-- if (folderIndex === -1) {- if (folderIds.length) {- return { done: false, cursor: startOfFolder(cursor.projectId, folderIds[0]), completedProjectId: null };- }- const next = firstProjectWithFolders(projectIds, folderIdsByProject, projectIndex + 1);- if (!next) return { done: true };- return { done: false, cursor: startOfFolder(next.projectId, next.folderId), completedProjectId: null };- }-- if (!folderExhausted && lastRow) {- return {- done: false,- cursor: { projectId: cursor.projectId, folderId: cursor.folderId, key: lastRow.key, id: lastRow.id },- completedProjectId: null- };- }-- if (folderIndex + 1 < folderIds.length) {- return {- done: false,- cursor: startOfFolder(cursor.projectId, folderIds[folderIndex + 1]),- completedProjectId: null- };- }-- const next = firstProjectWithFolders(projectIds, folderIdsByProject, projectIndex + 1);- if (!next) return { done: true };- return { done: false, cursor: startOfFolder(next.projectId, next.folderId), completedProjectId: cursor.projectId };-};--export const needsBackfill = (row: {- secretValueBlindIndex?: string | null;- secretValueOrgBlindIndex?: string | null;- encryptedValue?: Buffer | null;-}) => Boolean(row.encryptedValue) && (!row.secretValueBlindIndex || !row.secretValueOrgBlindIndex);-```--- [ ] **Step 5: Run the tests to verify they pass**--Run: `make test-api-unit SPEC=secret-value-tracking-fns`--Expected: PASS, 13 tests.--One test needs attention if it fails. "the last folder of the last project ends the walk" returns `{ done: true }` and therefore reports no `completedProjectId`, so the caller flips the final project's flag from the `done` branch rather than from the cursor. Task 5 does that.--- [ ] **Step 6: Commit**--```bash-git add backend/src/services/secret-value-tracking/-git commit -m "feat(secret-value-tracking): add the pure walk logic"-```-------### Task 3: Run state in Redis--**Files:**-- Modify: `backend/src/keystore/keystore.ts` (add the prefix)-- Create: `backend/src/services/secret-value-tracking/secret-value-tracking-state.ts`-- Modify: `backend/src/services/secret-value-tracking/secret-value-tracking-fns.ts` (add `resolveRunStatus`)-- Test: `backend/src/services/secret-value-tracking/secret-value-tracking-fns.test.ts` (extend)--**Interfaces:**-- Consumes: `TBackfillRunState`, `TBackfillCursor` from Task 2.-- Produces:- - `KeyStorePrefixes.SecretValueTrackingBackfill(scopeId: string): string`- - `resolveRunStatus(state: TBackfillRunState | null, flagEnabled: boolean, now: Date): { status: JobState; message?: string }`- - `secretValueTrackingStateFactory({ keyStore })` returning `{ read, claim, write, clear }` where- - `read(scopeId: string): Promise<TBackfillRunState | null>`- - `claim(scopeId: string, projectsTotal: number): Promise<boolean>`- - `write(scopeId: string, state: TBackfillRunState): Promise<void>`- - `clear(scopeId: string): Promise<void>`--- [ ] **Step 1: Add the key prefix and the constants**--In `backend/src/keystore/keystore.ts`, beside `InsightsCache` in `KeyStorePrefixes`:--```ts- SecretValueTrackingBackfill: (scopeId: string) => `secret-value-tracking-backfill:${scopeId}` as const,-```--And in `KeyStoreTtls`:--```ts- SecretValueTrackingBackfillInSeconds: 172800, // 2 days, long enough to resume a run someone retries the next day-```--- [ ] **Step 2: Write the failing tests for status resolution**--Append to `secret-value-tracking-fns.test.ts`, extending the existing import of the module under test and adding the state type:--```ts-import { JobState } from "@app/queue";--import { resolveRunStatus } from "./secret-value-tracking-fns";-import { TBackfillRunState } from "./secret-value-tracking-types";--const NOW = new Date("2026-09-24T12:00:00.000Z");-const minutesAgo = (n: number) => new Date(NOW.getTime() - n * 60_000).toISOString();--const running = (lastProgressAt: string): TBackfillRunState => ({- status: "running",- cursor: null,- projectsTotal: 3,- projectsDone: 1,- secretsProcessed: 10,- lastProgressAt-});--describe("resolveRunStatus", () => {- test("the flag being set means completed, whatever the key says", () => {- expect(resolveRunStatus(running(minutesAgo(1)), true, NOW)).toEqual({ status: JobState.Completed });- });-- test("no key and no flag means the backfill was never run", () => {- expect(resolveRunStatus(null, false, NOW)).toEqual({ status: JobState.NotFound });- });-- test("a run that progressed recently is pending", () => {- expect(resolveRunStatus(running(minutesAgo(1)), false, NOW)).toEqual({ status: JobState.Pending });- });-- test("a recorded failure reports its error", () => {- expect(- resolveRunStatus({ ...running(minutesAgo(1)), status: "failed", error: "kms unavailable" }, false, NOW)- ).toEqual({ status: JobState.Failed, message: "kms unavailable" });- });-- test("a run with no progress for longer than the window is stalled", () => {- const result = resolveRunStatus(running(minutesAgo(30)), false, NOW);- expect(result.status).toBe(JobState.Failed);- expect(result.message).toMatch(/stopped responding/i);- });-- // Review Focus 4: the boundary must be decisive rather than flapping.- test("exactly at the staleness boundary the run still counts as pending", () => {- expect(resolveRunStatus(running(minutesAgo(15)), false, NOW)).toEqual({ status: JobState.Pending });- });-- test("one second past the boundary the run is stalled", () => {- const justPast = new Date(NOW.getTime() - (15 * 60_000 + 1000)).toISOString();- expect(resolveRunStatus(running(justPast), false, NOW).status).toBe(JobState.Failed);- });-- // Review Focus 5: an unreadable lastProgressAt must not throw into the status endpoint.- test("an unparseable lastProgressAt reads as stalled rather than throwing", () => {- expect(resolveRunStatus({ ...running(""), lastProgressAt: "not-a-date" }, false, NOW).status).toBe(JobState.Failed);- });-});-```--- [ ] **Step 3: Run the tests to verify they fail**--Run: `make test-api-unit SPEC=secret-value-tracking-fns`--Expected: FAIL with `resolveRunStatus is not a function`.--- [ ] **Step 4: Implement status resolution**--Append to `secret-value-tracking-fns.ts`:--```ts-import { JobState } from "@app/queue";--import { TBackfillRunState } from "./secret-value-tracking-types";--export const BACKFILL_STALE_AFTER_MS = 15 * 60 * 1000;--export const resolveRunStatus = (- state: TBackfillRunState | null,- flagEnabled: boolean,- now: Date-): { status: JobState; message?: string } => {- if (flagEnabled) return { status: JobState.Completed };- if (!state) return { status: JobState.NotFound };- if (state.status === "failed") return { status: JobState.Failed, message: state.error ?? "Unknown error" };-- const lastProgress = new Date(state.lastProgressAt).getTime();- if (Number.isNaN(lastProgress) || now.getTime() - lastProgress > BACKFILL_STALE_AFTER_MS) {- return {- status: JobState.Failed,- message: "The backfill stopped responding. Start it again to resume from where it left off."- };- }-- return { status: JobState.Pending };-};-```--- [ ] **Step 5: Run the tests to verify they pass**--Run: `make test-api-unit SPEC=secret-value-tracking-fns`--Expected: PASS, 21 tests.--- [ ] **Step 6: Write the state module**--Create `secret-value-tracking-state.ts`. `read` must never throw on a bad payload, because it feeds the status endpoint and an unreadable key means "no run in progress", not a 500.--```ts-import { KeyStorePrefixes, KeyStoreTtls, TKeyStoreFactory } from "@app/keystore/keystore";-import { logger } from "@app/lib/logger";--import { BACKFILL_STALE_AFTER_MS } from "./secret-value-tracking-fns";-import { TBackfillRunState } from "./secret-value-tracking-types";--type TDep = {- keyStore: Pick<TKeyStoreFactory, "getItemPrimary" | "setItemWithExpiry" | "setItemWithExpiryNX" | "deleteItem">;-};--export const secretValueTrackingStateFactory = ({ keyStore }: TDep) => {- const read = async (scopeId: string): Promise<TBackfillRunState | null> => {- const raw = await keyStore.getItemPrimary(KeyStorePrefixes.SecretValueTrackingBackfill(scopeId));- if (!raw) return null;-- try {- const parsed = JSON.parse(raw) as TBackfillRunState;- if (typeof parsed?.lastProgressAt !== "string" || typeof parsed?.status !== "string") return null;- return parsed;- } catch (error) {- logger.warn(error, `Secret value tracking backfill state unreadable [scopeId=${scopeId}]`);- return null;- }- };-- const write = async (scopeId: string, state: TBackfillRunState) => {- await keyStore.setItemWithExpiry(- KeyStorePrefixes.SecretValueTrackingBackfill(scopeId),- KeyStoreTtls.SecretValueTrackingBackfillInSeconds,- JSON.stringify(state)- );- };-- const clear = async (scopeId: string) => {- await keyStore.deleteItem(KeyStorePrefixes.SecretValueTrackingBackfill(scopeId));- };-- // Claiming keeps whatever cursor a previous run left, so retrying a stalled backfill resumes- // instead of starting the scope again.- const claim = async (scopeId: string, projectsTotal: number) => {- const fresh: TBackfillRunState = {- status: "running",- cursor: null,- projectsTotal,- projectsDone: 0,- secretsProcessed: 0,- lastProgressAt: new Date().toISOString()- };-- const claimed = await keyStore.setItemWithExpiryNX(- KeyStorePrefixes.SecretValueTrackingBackfill(scopeId),- KeyStoreTtls.SecretValueTrackingBackfillInSeconds,- JSON.stringify(fresh)- );- if (claimed) return true;-- const existing = await read(scopeId);- if (!existing) {- await write(scopeId, fresh);- return true;- }-- const lastProgress = new Date(existing.lastProgressAt).getTime();- const stalled =- existing.status === "failed" ||- Number.isNaN(lastProgress) ||- Date.now() - lastProgress > BACKFILL_STALE_AFTER_MS;- if (!stalled) return false;-- await write(scopeId, {- ...fresh,- cursor: existing.cursor,- projectsDone: existing.projectsDone,- secretsProcessed: existing.secretsProcessed- });- return true;- };-- return { read, write, clear, claim };-};--export type TSecretValueTrackingStateFactory = ReturnType<typeof secretValueTrackingStateFactory>;-```--- [ ] **Step 7: Type check and commit**--Run: `cd backend && npm run type:check`--Expected: exit 0.--```bash-git add backend/src/keystore/keystore.ts backend/src/services/secret-value-tracking/-git commit -m "feat(secret-value-tracking): add the backfill run state"-```-------### Task 4: The keyset query and the relaxed update guard--**Files:**-- Modify: `backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts:1269-1284` (replace `findProjectSecretsWithNullBlindIndex`)-- Modify: `backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts:1305` (the update guard)--**Interfaces:**-- Consumes: nothing.-- Produces: `findSecretsInFolderAfter(folderId: string, after: { key: string; id: string }, limit: number, tx?: Knex): Promise<Pick<TSecretsV2, "id" | "key" | "encryptedValue" | "secretValueBlindIndex" | "secretValueOrgBlindIndex">[]>`, ordered by `key` then `id`.--- [ ] **Step 1: Replace the discovery query**--Delete `findProjectSecretsWithNullBlindIndex` and add:--```ts- // Walks a folder in (key, id) order so a backfill visits every row exactly once. Ordering on the- // pair matters: the unique index on (key, folderId) is partial on type = 'shared', so a personal- // override shares a key with the shared secret beside it and a key-only cursor would step over it.- const findSecretsInFolderAfter = async (- folderId: string,- after: { key: string; id: string },- limit: number,- tx?: Knex- ) => {- try {- const docs = await (tx || db.replicaNode())(TableName.SecretV2)- .where(`${TableName.SecretV2}.folderId`, folderId)- .whereRaw(`("${TableName.SecretV2}"."key", "${TableName.SecretV2}"."id"::text) > (?, ?)`, [after.key, after.id])- .orderBy([- { column: `${TableName.SecretV2}.key`, order: "asc" },- { column: `${TableName.SecretV2}.id`, order: "asc" }- ])- .limit(limit)- .select(- db.ref("id").withSchema(TableName.SecretV2),- db.ref("key").withSchema(TableName.SecretV2),- db.ref("encryptedValue").withSchema(TableName.SecretV2),- db.ref("secretValueBlindIndex").withSchema(TableName.SecretV2),- db.ref("secretValueOrgBlindIndex").withSchema(TableName.SecretV2)- );-- return docs as Pick<- TSecretsV2,- "id" | "key" | "encryptedValue" | "secretValueBlindIndex" | "secretValueOrgBlindIndex"- >[];- } catch (error) {- throw new DatabaseError({ error, name: "FindSecretsInFolderAfter" });- }- };-```--Export it from the factory's return object in place of `findProjectSecretsWithNullBlindIndex`.--The `id::text` cast is deliberate. `id` is a uuid and the cursor carries it as a string, so comparing the row constructor without the cast makes Postgres compare a uuid against text and fail. Casting on both sides of the comparison keeps the ordering consistent with the `ORDER BY`, which also sorts uuid as text under this cast, so the walk stays total.--- [ ] **Step 2: Relax the update guard**--At `secret-v2-bridge-dal.ts:1305`, change the trailing condition of the `batchSetBlindIndexes` raw query:--```ts- WHERE ${TableName.SecretV2}.id = v.id- AND (${TableName.SecretV2}."secretValueBlindIndex" IS NULL- OR ${TableName.SecretV2}."secretValueOrgBlindIndex" IS NULL)-```--The guard exists to avoid overwriting a row a live write changed between the backfill's read and its write. Every live write sets both digests, so the relaxed guard still skips any row a live write has touched; the only rows it now lets through are ones written before one of the digests existed, and nothing is concurrently writing those.--- [ ] **Step 3: Fix the one caller so the build passes**--`backend/src/services/project/project-queue.ts` references `findProjectSecretsWithNullBlindIndex` in its dependency `Pick` and in the job body. Task 5 removes that job entirely. For this task, narrow the `Pick` to `"batchSetBlindIndexes"` and leave the job body compiling by deleting the `findProjectSecretsWithNullBlindIndex` call together with the job, which Task 5 then rebuilds elsewhere.--If that is too large a step to keep the build green in one commit, do Task 4 and Task 5 as a single commit. They are split here only because their tests differ.--- [ ] **Step 4: Type check**--Run: `cd backend && npm run type:check`--Expected: exit 0.--- [ ] **Step 5: Commit**--```bash-git add backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts backend/src/services/project/project-queue.ts-git commit -m "refactor(secret-value-tracking): walk secrets by cursor instead of scanning for nulls"-```-------### Task 5: The chunked job--**Files:**-- Create: `backend/src/services/secret-value-tracking/secret-value-tracking-queue.ts`-- Modify: `backend/src/queue/queue-service.ts:623-626` (the payload)-- Modify: `backend/src/services/project/project-queue.ts` (remove the job, `startSecretBlindIndexMigration`, `getJobState`)--**Interfaces:**-- Consumes: `advanceCursor`, `needsBackfill`, `TBackfillScope`, `TBackfillCursor` (Task 2), `secretValueTrackingStateFactory` (Task 3), `findSecretsInFolderAfter`, `batchSetBlindIndexes` (Task 4).-- Produces: `secretValueTrackingQueueFactory(dep)` returning `{ startBackfill(scope: TBackfillScope): Promise<void> }`. It takes no project count, because the service has already written that into the Redis key when it claimed the run.--- [ ] **Step 1: Widen the queue payload**--In `backend/src/queue/queue-service.ts`:--```ts- [QueueName.SecretBlindIndexMigration]: {- name: QueueJobs.SecretBlindIndexMigration;- // `scope` is absent on jobs queued before the org-wide walk shipped; the handler reads those as- // project scope so a deploy does not strand them.- payload: { scope: "org"; orgId: string } | { scope: "project"; projectId: string } | { projectId: string };- };-```--- [ ] **Step 2: Write the job**--Create `secret-value-tracking-queue.ts`. The factory dependencies are:--```ts-type TDep = {- queueService: TQueueServiceFactory;- keyStore: Pick<TKeyStoreFactory, "getItemPrimary" | "setItemWithExpiry" | "setItemWithExpiryNX" | "deleteItem" | "deleteItems">;- projectDAL: Pick<TProjectDALFactory, "find" | "findById" | "updateById">;- orgDAL: Pick<TOrgDALFactory, "updateById">;- folderDAL: Pick<TSecretFolderDALFactory, "findByProjectId">;- secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "findSecretsInFolderAfter" | "batchSetBlindIndexes">;- kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;-};-```--Constants:--```ts-const CHUNK_SIZE = 5000;-const READ_BATCH_SIZE = 1000;-```--Behaviour, in order:--1. Read the scope from the payload. A payload with no `scope` is project scope.-2. `scopeId` is the `orgId` for org scope and the `projectId` for project scope. It keys the Redis state.-3. Read the run state. If there is none, stop: the run was cleared or expired, and a chunk with no state has nothing to resume.-4. Resolve the project list. For org scope, `projectDAL.find({ orgId, type: ProjectType.SecretManager })` sorted by `id`. For project scope, the single project, and `orgId` comes from it.-5. Resolve the folder list per project with `folderDAL.findByProjectId`, sorted by `id`. Fetch it lazily, only for the projects the walk touches in this chunk.-6. Loop until `CHUNK_SIZE` secrets have been read or the walk is done:- - Resolve the current cursor with `advanceCursor` when there is none, or use the stored one.- - Read up to `READ_BATCH_SIZE` rows with `findSecretsInFolderAfter(cursor.folderId, { key: cursor.key, id: cursor.id }, READ_BATCH_SIZE)`.- - `folderExhausted` is `rows.length < READ_BATCH_SIZE`.- - Filter with `needsBackfill`, decrypt each survivor, compute both digests with the indexer, and call `batchSetBlindIndexes`.- - Advance the cursor with `advanceCursor`, passing the last row read.- - When the result carries a `completedProjectId`, set `projectDAL.updateById(completedProjectId, { secretBlindIndexEnabled: true })` and increment `projectsDone`.-7. The cipher pair is resolved once per project encountered, not once per batch, with `createSecretBlindIndexer({ projectId, orgId, kmsService })`. Cache it in a `Map` keyed by `projectId` for the life of the chunk. This is why the chunk is 5000 rather than the read batch size: on external KMS each resolve is a network call.-8. On `done`, flip the final project's flag too, then for org scope set `orgDAL.updateById(orgId, { orgWideSecretValueTrackingEnabled: true })` and for project scope set the project's own flag. Clear the Redis key. Delete the insights duplication cache with `keyStore.deleteItems({ pattern: `${KeyStorePrefixes.InsightsCache(projectId, "secrets-duplication")}*` })` for each project the walk completed, which is what the job does today.-9. Otherwise write the advanced state with a fresh `lastProgressAt` and queue the successor.--The successor's job id must be unique per chunk, because BullMQ ignores an add for an id that is currently active, so a fixed id would break the chain at the second chunk:--```ts- const jobId = `secret-value-tracking-${scopeId}-${Date.now()}`;-```--Register the `failed` listener so an exhausted retry is visible on the status endpoint rather than only in the logs:--```ts- queueService.listen(QueueName.SecretBlindIndexMigration, "failed", (job, err) => {- const scopeId = resolveScopeId(job?.data);- logger.error(err, `SecretValueTrackingBackfill: failed [scopeId=${scopeId}]`);- if (!scopeId) return;- void state.read(scopeId).then((existing) => {- if (!existing) return undefined;- return state.write(scopeId, { ...existing, status: "failed", error: err.message });- });- });-```--Start the queue with the pacing on the queue rather than in the loop, and drop the 100 to 200ms sleep the old job used between batches:--```ts- queueService.start(QueueName.SecretBlindIndexMigration, handler, {- concurrency: 1,- limiter: { max: 1, duration: 1000 }- });-```--`startBackfill(scope)` queues the first chunk. It does not claim the Redis key; the service does that before calling, so a refused claim never queues anything.--A project or folder that vanished is handled entirely by `advanceCursor`, so the job body needs no special case for it.--- [ ] **Step 3: Remove the old job**--Delete `startSecretBlindIndexMigration`, the `queueService.start(QueueName.SecretBlindIndexMigration, ...)` block, its `failed` listener, and `getJobState` from `project-queue.ts`, along with the now unused imports (`createSecretBlindIndexer`, `KmsDataKey` if unused elsewhere in that file, `JobState`). Remove them from the factory's return object. `queueService.start` must be called once per queue name, so leaving both would register two workers on one queue.--- [ ] **Step 4: Type check**--Run: `cd backend && npm run type:check`--Expected: FAIL, naming `project-service.ts` where `projectQueue.startSecretBlindIndexMigration` and `projectQueue.getJobState` no longer exist. Task 6 fixes those call sites.--- [ ] **Step 5: Commit**--Commit with Task 6, since the build is not green on its own.-------### Task 6: The service, the routes, and the wiring--**Files:**-- Create: `backend/src/services/secret-value-tracking/secret-value-tracking-service.ts`-- Modify: `backend/src/services/project/project-service.ts:2361-2409` (delegate)-- Modify: `backend/src/server/routes/v1/organization-router.ts` (two routes)-- Modify: `backend/src/server/routes/index.ts` (wiring)-- Modify: `backend/src/ee/services/audit-log/audit-log-types.ts` (the event)-- Modify: `backend/src/@types/fastify.d.ts` (the service on `server.services`)--**Interfaces:**-- Consumes: `secretValueTrackingQueueFactory` (Task 5), `secretValueTrackingStateFactory` and `resolveRunStatus` (Task 3).-- Produces: `secretValueTrackingServiceFactory(dep)` returning- - `enableForOrg(actor: OrgServiceActor): Promise<{ projectsTotal: number }>` (the count travels back so the route can record it on the audit event)- - `getOrgStatus(actor: OrgServiceActor): Promise<{ status: JobState; message?: string; projectsTotal: number; projectsDone: number; secretsProcessed: number }>`- - `enableForProject(dto: TProjectPermission): Promise<void>`- - `getProjectStatus(dto: TProjectPermission): Promise<{ status: JobState; message?: string; projectsTotal: number; projectsDone: number; secretsProcessed: number }>`--- [ ] **Step 1: Write the service**--`enableForOrg`:-1. `ForbiddenError.from(permission).throwUnlessCan(OrgPermissionActions.Edit, OrgPermissionSubjects.Settings)` using `permissionService.getOrgPermission`.-2. Read the org with `orgDAL.findById(actor.orgId)`. Throw `NotFoundError` when missing.-3. Throw `BadRequestError` with `"Org-wide secret value tracking is already enabled for this organization"` when the flag is set, matching how `enableSecretBlindIndex` refuses.-4. Count projects: `(await projectDAL.find({ orgId: actor.orgId, type: ProjectType.SecretManager })).length`. Do not count secrets.-5. `const claimed = await state.claim(actor.orgId, projectsTotal)`. When false, throw `BadRequestError` with `"A backfill is already running for this organization"`.-6. `await queue.startBackfill({ scope: "org", orgId: actor.orgId })` and return `{ projectsTotal }`.--`getOrgStatus` reads the org flag and the state, returns `resolveRunStatus(state, org.orgWideSecretValueTrackingEnabled, new Date())` spread with `projectsTotal`, `projectsDone` and `secretsProcessed` from the state, defaulting to `0` when there is no state.--`enableForProject` and `getProjectStatus` keep the permission checks that `project-service.ts` uses today (`ProjectPermissionActions.Edit` on `ProjectPermissionSub.Settings` via `getProjectPermission` with `actionProjectType: ActionProjectType.SecretManager`), gate on `projects.secretBlindIndexEnabled`, and use the project id as the scope id. `projectsTotal` is `1`.--- [ ] **Step 2: Add the audit event**--In `backend/src/ee/services/audit-log/audit-log-types.ts`, beside `UPDATE_ORG`:--```ts- ENABLE_ORG_WIDE_SECRET_VALUE_TRACKING = "enable-org-wide-secret-value-tracking",-```--```ts-interface EnableOrgWideSecretValueTrackingEvent {- type: EventType.ENABLE_ORG_WIDE_SECRET_VALUE_TRACKING;- metadata: {- projectsTotal: number;- };-}-```--Add `| EnableOrgWideSecretValueTrackingEvent` to the `Event` union.--- [ ] **Step 3: Write the routes**--In `backend/src/server/routes/v1/organization-router.ts`, which is registered under the `/organization` prefix:--```ts- server.route({- method: "POST",- url: "/secret-value-tracking",- config: { rateLimit: writeLimit },- schema: {- hide: true,- operationId: "enableOrgWideSecretValueTracking",- description: "Start the backfill that makes every secret value in the organization searchable",- security: [{ bearerAuth: [] }],- response: { 200: z.object({ message: z.string() }) }- },- onRequest: verifyAuth([AuthMode.JWT]),- handler: async (req) => {- const { projectsTotal } = await server.services.secretValueTracking.enableForOrg(req.permission);-- await server.services.auditLog.createAuditLog({- ...req.auditLogInfo,- orgId: req.permission.orgId,- event: {- type: EventType.ENABLE_ORG_WIDE_SECRET_VALUE_TRACKING,- metadata: { projectsTotal }- }- });-- return { message: "Successfully started org-wide secret value tracking" };- }- });-- server.route({- method: "GET",- url: "/secret-value-tracking/status",- config: { rateLimit: readLimit },- schema: {- hide: true,- operationId: "getOrgWideSecretValueTrackingStatus",- description: "Report progress of the org-wide secret value tracking backfill",- security: [{ bearerAuth: [] }],- response: {- 200: z.object({- status: z.nativeEnum(JobState),- message: z.string().optional(),- projectsTotal: z.number(),- projectsDone: z.number(),- secretsProcessed: z.number()- })- }- },- onRequest: verifyAuth([AuthMode.JWT, AuthMode.OAUTH]),- handler: async (req) => server.services.secretValueTracking.getOrgStatus(req.permission)- });-```--The enable route is `AuthMode.JWT` only. It is an organization settings write, which `backend/CLAUDE.md` says does not take a delegated OAuth token; the status read does.--- [ ] **Step 4: Delegate the project path**--In `project-service.ts`, replace the bodies of `enableSecretBlindIndex` and `getSecretBlindIndexMigrationStatus` with calls to `secretValueTrackingService.enableForProject` and `.getProjectStatus`. Keep the exported names so `project-router.ts` is untouched.--- [ ] **Step 5: Wire it**--In `backend/src/server/routes/index.ts`, construct the state, queue, and service factories after `secretV2BridgeDAL` and `folderDAL` exist, pass `secretValueTrackingService` into `projectServiceFactory`, and add `secretValueTracking: secretValueTrackingService` to the `server.decorate("services", {...})` object. Declare it on `TSecretValueTrackingServiceFactory` in `backend/src/@types/fastify.d.ts`.--- [ ] **Step 6: Type check and lint**--Run: `cd backend && npm run type:check && npm run lint:fix`--Expected: both exit 0.--- [ ] **Step 7: Commit**--```bash-git add backend/-git commit -m "feat(secret-value-tracking): backfill a whole org from one chunked job"-```-------### Task 7: Gate secret value search--**Files:**-- Modify: `backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts` (`findSecretsByValue`)--**Interfaces:**-- Consumes: `organizations.orgWideSecretValueTrackingEnabled` (Task 1).-- Produces: nothing new.--- [ ] **Step 1: Add the gate**--`findSecretsByValue` already takes `actor: OrgServiceActor` and the factory already has `orgDAL` with `findOrgById`. Add `"findById"` to that `Pick` and read the org first, before any other work:--```ts- const org = await orgDAL.findById(actor.orgId);- if (!org) throw new NotFoundError({ message: `Organization with ID '${actor.orgId}' not found` });-- if (!org.orgWideSecretValueTrackingEnabled) {- throw new BadRequestError({- message:- "Enable org-wide secret value tracking for this organization before searching for a secret by its value"- });- }-```--Use `findById` rather than `findOrgById`: `findOrgById` takes no `tx` and reads from `db.replicaNode()`, which is the deadlock trigger `CODE_QUALITY.md` warns about if this ever moves inside a transaction.--Both scopes are gated, project scope included, because both match on the organization digest.--- [ ] **Step 2: Type check**--Run: `cd backend && npm run type:check`--Expected: exit 0.--- [ ] **Step 3: Commit**--```bash-git add backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts-git commit -m "feat(secret-value-tracking): gate secret value search on org completion"-```-------### Task 8: End to end--**Files:**-- Create: `backend/e2e-test/routes/v1/org-secret-value-tracking.spec.ts`-- Modify: `backend/e2e-test/routes/v3/secrets-management.spec.ts` (enable tracking before the search tests)--**Interfaces:**-- Consumes: everything above.-- Produces: nothing.--- [ ] **Step 1: Unblock the existing search suite**--`createIsolatedOrgAndProject` makes a fresh organization, which the migration defaults to tracking enabled, so `secrets-management.spec.ts` should keep passing untouched. Run it first to confirm:--Run: `make test-api-e2e SPEC=secrets-management`--Expected: PASS, 11 tests. If it fails with the gate's message, the migration's default for new rows is wrong and Task 1 needs revisiting before going further.--- [ ] **Step 2: Write the failing e2e spec**--Create `org-secret-value-tracking.spec.ts`. It uses `createIsolatedOrgAndProject` from `e2e-test/testUtils/fixtures`, the secret helpers from `e2e-test/testUtils/secrets`, and `testServer.inject`.--Cover, in this order:--```ts-const enable = (authToken: string) =>- testServer.inject({- method: "POST",- url: "/api/v1/organization/secret-value-tracking",- headers: { authorization: `Bearer ${authToken}` }- });--const status = async (authToken: string) => {- const res = await testServer.inject({- method: "GET",- url: "/api/v1/organization/secret-value-tracking/status",- headers: { authorization: `Bearer ${authToken}` }- });- expect(res.statusCode).toBe(200);- return res.json() as { status: string; projectsTotal: number; projectsDone: number; secretsProcessed: number };-};-```--- "a new organization is already tracking" asserts `status` is `completed` and that `enable` returns 400 with the already-enabled message.-- "a search refuses while tracking is off" flips the flag off directly through the org update route or a DAL call in the test setup, asserts the search returns 400 naming org-wide secret value tracking, then re-enables and asserts the search works again.-- "the backfill fills a secret whose org digest was cleared" is the case the whole feature exists for. Create a secret, null its `secretValueOrgBlindIndex` and the org flag, run `enable`, poll `status` until `completed` (with a timeout well inside the suite's `vi.setConfig` budget), then assert the value is findable through `POST /api/v4/secrets/search-by-value`.-- "a second enable while a run is moving is refused" asserts the 400.--Writing to the database directly is acceptable here only for *setup*, to manufacture the incomplete state that a fresh test organization cannot otherwise have. Every assertion goes through the API.--Set `vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 })` at the top of the describe, because the walk is a queue round trip.--- [ ] **Step 3: Run it to verify it fails**--Run: `make test-api-e2e SPEC=org-secret-value-tracking`--Expected: FAIL on the first assertion that exercises something not yet correct.--- [ ] **Step 4: Fix what it finds, then run again**--Run: `make test-api-e2e SPEC=org-secret-value-tracking`--Expected: PASS.--- [ ] **Step 5: Run the wider suites that touch this code**--Run: `make test-api-unit SPEC=secret-value-tracking && make test-api-e2e SPEC=secrets-management && make test-api-e2e SPEC=folder`--Expected: all PASS. `folder` covers the rollback path that copies digests off version rows, and `secrets-management` covers the search.--- [ ] **Step 6: Run the full review gate**--Run: `make reviewable-api`--Expected: exit 0.--- [ ] **Step 7: Commit**--```bash-git add backend/e2e-test/-git commit -m "test(secret-value-tracking): cover the org backfill end to end"-```-------## Notes for the implementer--- `backend/CODE_QUALITY.md` applies to every task. The two items most likely to bite here are transaction discipline (the walk must never hold a transaction across a KMS call) and error messages a user can act on.-- The walk reads rows that already carry both digests and skips them in code. That is intentional and is not a bug to optimise away: it is what lets the walk use an index that already exists instead of adding one whose only purpose is the migration.-- Do not add a Postgres index for the backfill. That was considered and rejected, because a user-triggered backfill never tells us when it is safe to drop the index again.diff --git a/docs/superpowers/specs/2026-09-24-org-wide-secret-value-tracking-backfill-design.md b/docs/superpowers/specs/2026-09-24-org-wide-secret-value-tracking-backfill-design.mddeleted file mode 100644index bf9deb83a3..0000000000--- a/docs/superpowers/specs/2026-09-24-org-wide-secret-value-tracking-backfill-design.md+++ /dev/null@@ -1,284 +0,0 @@-# Org-wide secret value tracking backfill--## Context--`secrets_v2` carries two digests of a secret's value. `secretValueBlindIndex` is keyed by the-project's KMS data key, so it can only answer questions inside one project.-`secretValueOrgBlindIndex` is keyed by the org's data key, so equal values anywhere in an org-produce equal digests. Every writer computes both on every create and update.--Neither column is filled for rows written before its digest existed. Projects have a-user-triggered backfill for the project digest (`startSecretBlindIndexMigration` in-`backend/src/services/project/project-queue.ts`), gated behind-`projects.secretBlindIndexEnabled`, and the product asks the user to enable it when they reach-for a feature that needs it (duplicate detection in insights, and reuse rules in-`secret-validation-rule-service.ts`).--Secret value search (`POST /api/v4/secrets/search-by-value`) needs the same treatment one level-up. It matches on the org digest, so it can only be trusted once every row in the org has one.-Answering "this value is used nowhere" from a half-filled index is the worst possible answer for-someone chasing a leaked credential, so the feature has to be blocked until the org is complete-rather than allowed to return a partial result.--This design repurposes the project backfill into one job that can walk either scope, adds the-org-level enable and status endpoints to sit beside the project pair, and gates the search on-org completion.--## What we are building--1. One walking job that backfills both digests, over either a single project or a whole org.-2. An org-level enable endpoint and a status endpoint, mirroring the project pair.-3. `organizations.orgWideSecretValueTrackingEnabled`, the durable flag the search gates on.-4. A Redis key holding the run's transient state: status, cursor, progress, and last progress- time.-5. Secret value search refusing with a message naming what to enable, until that flag is set.--## The walk--### One predicate, two scopes--The job's unit of work is "this row is missing at least one digest". That single predicate is-what lets one job body serve both scopes, because we always compute both digests anyway and the-scope is only a filter on which rows we visit.--Two places have to change for that to be true, and skipping either one makes the org backfill a-no-op on exactly the projects it exists for:--- The row is worth working on when `secretValueBlindIndex IS NULL OR secretValueOrgBlindIndex IS- NULL`. A project that already ran the project backfill has a project digest on every row and no- org digest, so a predicate that only looks at the project digest finds nothing there.-- The update guard in `batchSetBlindIndexes` (`secret-v2-bridge-dal.ts`) has to match, for the- same reason. It currently guards `AND "secretValueBlindIndex" IS NULL`.--Relaxing that guard is safe. It exists to avoid clobbering a row that a live write changed-between our read and our write, and every live write sets both digests, so the relaxed guard-still skips any row a live write has touched. The only rows that match are ones written by code-that predates one of the digests, and nothing is concurrently writing those.--### Keyset pagination over an index we already have--`findProjectSecretsWithNullBlindIndex` asks "give me rows in this project with no digest, limit-N". Postgres can use the leading column of `idx_secrets_v2_folder_id_key` to reach the project's-rows, but the `IS NULL` filter has no index behind it, so it is applied per row after the fetch.-That is tolerable once. The problem is that the job re-runs the same query for every batch, so-each batch re-reads and discards everything the previous batches already filled, and the last-batch reads most of the project to find a handful of rows.--We replace it with a cursor walk over the index that already exists:--```sql-SELECT id, "encryptedValue", "secretValueBlindIndex", "secretValueOrgBlindIndex"-FROM secrets_v2-WHERE "folderId" = ? AND ("key", id) > (?, ?)-ORDER BY "key", id-LIMIT ?-```--Every row is visited exactly once for the whole backfill, through an index range scan, and no new-Postgres index is needed. Rows that already carry both digests are skipped in code before we-decrypt anything, so the only cost they add is the read.--Ordering on `(key, id)` rather than `key` alone is load-bearing. The unique index on-`(key, folderId)` is partial (`WHERE type = 'shared'`), so a personal override shares a key with-the shared secret in the same folder, and a key-only cursor would step over it.--The walk nests three levels:--- Projects in the org, from `projectDAL.find({ orgId, type: SecretManager })`, ordered by id.-- Folders in the project, from `folderDAL.findByProjectId`, ordered by id.-- Rows in the folder, by the keyset query above.--The cursor is `{ projectId, folderId, key, id }`. When a folder runs out we move to the next-folder id, when a project runs out we flip that project's `secretBlindIndexEnabled` and move to-the next project, and when the projects run out the run is complete.--### Chunking--One job processes up to `CHUNK_SIZE` secrets (5000, read in database batches of 1000), writes the-advanced cursor to Redis, and queues its successor. If there is nothing left it sets the durable-flag and deletes the key instead.--Chunking at 5000 rather than at the database batch size is about the KMS round trips.-`createSecretBlindIndexer` resolves a project data key and an org data key once per job, and for-an org on external KMS each of those is a network call, so a job per 1000 rows would pay that-five times as often for no benefit.--The queue runs at `concurrency: 1` with a limiter. That, rather than job length, is what keeps a-long backfill from crowding out other work: the walk can never hold more than one worker slot no-matter how many secrets an org has.--The existing queue name stays. The payload widens to a discriminated union:--```ts-{ scope: "org"; orgId: string } | { scope: "project"; projectId: string }-```--A payload arriving with no `scope` is treated as project scope, so jobs already queued in the old-shape survive the deploy.--## State--### The durable flag--`organizations.orgWideSecretValueTrackingEnabled`, boolean, not null.--The migration follows the one that added the project flag-(`20260604100000_add-secret-blind-index-enabled-to-project.ts`): default true for new rows, then-one update setting every existing org to false. New orgs are complete by construction, since-every secret they ever write carries both digests, so they should never be asked to run a-backfill.--The name deliberately does not say "blind index". The org schema should describe the capability-rather than the mechanism, which is the point lobocv raised in review on #8278.--### The Redis key--`KeyStorePrefixes.OrgSecretValueTrackingBackfill(orgId)`, holding JSON:--```ts-{- status: "running" | "failed";- cursor: { projectId: string; folderId: string; key: string; id: string } | null;- projectsTotal: number;- projectsDone: number;- secretsProcessed: number;- lastProgressAt: string; // ISO- error?: string;-}-```--There is no `completed` status, because completion is the durable flag and we do not want two-answers to the same question. The final chunk sets the flag and deletes the key.--The key has a 2 day TTL, refreshed on every chunk. The long TTL is for the cursor: someone who-notices a failure the next morning can retry and resume where the run stopped, rather than start-the org again.--`projectsTotal` is counted once when the run is enabled. We deliberately do not count secrets.-Counting them is itself an expensive read on exactly the orgs where a progress bar would matter,-and the number is stale the moment we have it, so progress reads as "project 3 of 12, 47,000-secrets processed" instead of a percentage. Projects created or deleted mid-walk make-`projectsDone / projectsTotal` drift a little, which is fine for a progress display.--### Why the guard reads lastProgressAt, not the key--The key does three jobs, and they want different timeouts. The cursor and the status want the-2 days. The guard that stops a second run starting wants a short window, because a worker that-died would otherwise lock the org out of retrying for two days, which is the opposite of what-somebody staring at a stuck progress bar wants.--So the guard reads `lastProgressAt`. Progress inside the last 15 minutes means a run is genuinely-moving and the enable is refused. Anything older is treated as dead, and the enable takes over,-resuming from the cursor still in the key.--That window only covers a worker disappearing without BullMQ noticing (eg a pod killed-mid-chunk). The ordinary failure is faster: the queue's `failed` listener writes-`status: "failed"` and the error into the key, so the status endpoint reports it immediately.--## API--Both routes sit in `backend/src/server/routes/v1/organization-router.ts`, beside the project pair-in `project-router.ts`.--### Enable--`POST /api/v1/organization/secret-value-tracking`--Requires `OrgPermissionActions.Edit` on `OrgPermissionSubjects.Settings`. Refuses with a-`BadRequestError` when the flag is already set, matching `enableSecretBlindIndex`. Counts the-org's projects, claims the Redis key, queues the first chunk, and writes an audit event.--Claiming the key is `setItemWithExpiryNX` when there is no key, and a plain overwrite that keeps-the existing cursor when the key is there but stale. Two people pressing the button at the same-instant could both get through the stale branch, which costs a duplicated chunk and nothing else,-since the writes are idempotent and the guard then holds for the rest of the run.--### Status--`GET /api/v1/organization/secret-value-tracking/status`--Returns `{ status, projectsTotal, projectsDone, secretsProcessed, message? }` where status is the-existing `JobState` enum. It is resolved without touching BullMQ:--- flag set, so `Completed`-- key present and `status: "failed"`, so `Failed` with the error-- key present and `lastProgressAt` within 15 minutes, so `Pending`-- key present and older than that, so `Failed` with a stalled message-- no key and no flag, so `NotFound`--This is also worth carrying back to the project pair eventually, whose status today is only as-durable as Redis and answers from BullMQ job state. That is out of scope here.--## Gating secret value search--`findSecretsByValue` in `secret-v2-bridge-service.ts` reads the org flag first and throws when it-is not set:--```-Enable org-wide secret value tracking for this organization before searching for a secret by its-value-```--Both scopes are gated on the org flag, including project scope, because both match on the org-digest.--This follows `$assertBlindIndexingAvailable` in `secret-validation-rule-service.ts`, which is how-the product already tells someone that a feature needs a backfill they have not run.--## What changes in existing code--- `secret-v2-bridge-dal.ts`: replace `findProjectSecretsWithNullBlindIndex` with the keyset query- above, and relax the guard in `batchSetBlindIndexes`.-- `project-queue.ts`: the job body becomes the walk, driven by the cursor. The project path uses- the same walk with the project-scoped payload, so the repeated scan goes away for projects too.- The 100 to 200ms pause between batches goes, since the queue limiter shapes the load now.-- `project-service.ts`: `enableSecretBlindIndex` keeps its shape. The new org service method sits- next to it.-- `secret-v2-bridge-service.ts`: the gate in `findSecretsByValue`.-- A migration for the org column.--The piece worth extracting for its own sake is cursor advancement. Deciding "given this cursor,-this project list, this folder list and the rows we just read, where does the walk go next" is-pure, and pulling it out of the job body is what makes it testable without a database, including-the awkward cases (a folder emptied since the last chunk, a project deleted mid-walk, the last-folder of the last project).--## Failure and recovery--- A chunk throws, so BullMQ retries it. The cursor is in Redis rather than the payload, so the- retry resumes at the same place instead of redoing work.-- The retries are exhausted, so the `failed` listener records the error in the key. The user sees- it on the status endpoint and can retry, which resumes from the cursor.-- The worker dies, so nothing records anything. The key goes stale, the status endpoint reports- the run as stalled after 15 minutes, and the enable endpoint accepts a retry that resumes from- the cursor.-- The key expires after 2 days, so the org reads as never enabled and a retry restarts the walk- from the first project. That costs a read pass, not re-work, because filled rows are skipped- before we decrypt.-- A project or folder in the cursor is deleted between chunks, so the walk steps past it and- carries on. It must not throw.--Secrets written during a run never need the walk to find them, because live writes already carry-both digests. So a row inserted behind the cursor being skipped is correct, not a hole.--## Testing--- Unit, on the extracted cursor advancement: moving through folders, moving through projects,- a deleted project or folder, an emptied folder, and the end of the walk.-- Unit, on the status resolution: each of the five states above, including the staleness- boundary.-- E2E: enable an org, poll status to completion, confirm search works afterwards and refuses with- the named message before. Cover a project that already ran the project backfill, which is the- case the predicate change exists for, and confirm the walk gives it org digests.-- E2E: a second enable while a run is moving is refused, and one after the staleness window is- accepted.--## Out of scope--- The UI for either endpoint, and the search UI that will call them.-- Reworking the project status endpoint to read from Redis rather than BullMQ, though the org one- shows what it would look like.-- Caching the org KMS data key, which is the larger performance item already noted on #8278.-- Personal secrets keep being given digests, as they are today, even though search filters them- out. Changing that is a separate decision.-- The write-path trim on `secretValue`, which means a padded value is stored and found trimmed.diff --git a/frontend/src/components/secrets/CheckedAtButton.tsx b/frontend/src/components/secrets/CheckedAtButton.tsxdeleted file mode 100644index 08a4e28eae..0000000000--- a/frontend/src/components/secrets/CheckedAtButton.tsx+++ /dev/null@@ -1,49 +0,0 @@-import { useEffect, useState } from "react";-import { formatDistanceToNow } from "date-fns";-import { RefreshCwIcon } from "lucide-react";--import { Badge } from "@app/components/v3";-import { cn } from "@app/components/v3/utils";--// Keeps "Checked N minutes ago" honest while the page sits open.-const useNow = (intervalMs: number) => {- const [now, setNow] = useState(() => Date.now());- useEffect(() => {- const id = setInterval(() => setNow(Date.now()), intervalMs);- return () => clearInterval(id);- }, [intervalMs]);- return now;-};--// Says how fresh a cached answer is and recomputes it on click, for cards whose result the server-// caches and nothing invalidates when the underlying data changes.-export const CheckedAtButton = ({- computedAt,- isRefreshing,- onRefresh-}: {- computedAt: string;- isRefreshing: boolean;- onRefresh: () => void;-}) => {- const now = useNow(30_000);- const computed = new Date(computedAt);- const label =- now - computed.getTime() < 60_000- ? "Checked just now"- : `Checked ${formatDistanceToNow(computed, { addSuffix: true })}`;-- return (- <Badge variant="neutral" asChild className="ml-2 font-normal">- <button- type="button"- onClick={onRefresh}- disabled={isRefreshing}- aria-label={`${label}. Check again`}- >- <RefreshCwIcon className={cn(isRefreshing && "animate-spin")} />- {isRefreshing ? "Checking..." : label}- </button>- </Badge>- );-};diff --git a/frontend/src/components/secrets/SecretValueSearchSheet.tsx b/frontend/src/components/secrets/SecretValueSearchSheet.tsxindex 565598638f..37b78c4312 100644--- a/frontend/src/components/secrets/SecretValueSearchSheet.tsx+++ b/frontend/src/components/secrets/SecretValueSearchSheet.tsx@@ -196,7 +196,7 @@ const SearchContent = ({ orgId, onClose }: { orgId: string; onClose: () => voidvalueAlwaysHiddenisVisible={isRevealed}containerClassName="flex-1"- placeholder="Paste a secret value..."+ placeholder="Paste the full secret value..."value={value}onChange={(e) => {setValue(e.target.value);@@ -235,7 +235,9 @@ export const SecretValueSearchSheet = ({ orgId, isOpen, onOpenChange }: Props) =<SheetHeader><SheetTitle>Search by Secret Value</SheetTitle><SheetDescription>- Find every project, environment and path where a secret value is used.+ Find every project, environment and path where a secret value is used. Only the full value+ matches, exactly as stored: partial values and values differing in whitespace are not+ found.</SheetDescription></SheetHeader>{/* Mounted only while open, so the value and results are dropped the moment it closes. */}diff --git a/frontend/src/components/secrets/SecretValueTrackingGate.tsx b/frontend/src/components/secrets/SecretValueTrackingGate.tsxindex 6552097789..2b79d22724 100644--- a/frontend/src/components/secrets/SecretValueTrackingGate.tsx+++ b/frontend/src/components/secrets/SecretValueTrackingGate.tsx@@ -1,4 +1,3 @@-import { useEffect, useRef, useState } from "react";import { AlertTriangleIcon, Loader2Icon } from "lucide-react";import {@@ -20,7 +19,6 @@ type TOrgSecretValueTracking = {isLoading: boolean;isTrackingOn: boolean;status?: TGetOrgSecretValueTrackingStatusResponse;- isRunStarting: boolean;isEnablePending: boolean;enable: () => void;};@@ -29,53 +27,21 @@ type TOrgSecretValueTracking = {// finished first, so they share one way of asking for it and of showing the run.export const useOrgSecretValueTracking = ({orgId,- enabled,- onTrackingOn+ enabled}: {orgId: string;enabled: boolean;- onTrackingOn?: () => void;}): TOrgSecretValueTracking => {const enableTracking = useEnableOrgSecretValueTracking();- const {- data: status,- isPending,- dataUpdatedAt- } = useGetOrgSecretValueTrackingStatus(orgId, { enabled });-- // Only a run this view started or saw in flight should report completion: a completed status- // seen on first load is simply "already on". Keyed on the fetch time because a small org can go- // from enable to completed before the first poll, so the status value alone may never change.- const [awaitingRun, setAwaitingRun] = useState(false);- const onTrackingOnRef = useRef(onTrackingOn);- onTrackingOnRef.current = onTrackingOn;- useEffect(() => {- if (status?.status === "pending") setAwaitingRun(true);- if (status?.status === "failed") setAwaitingRun(false);- if (status?.status === "completed" && awaitingRun) {- setAwaitingRun(false);- onTrackingOnRef.current?.();- }- }, [status?.status, dataUpdatedAt, awaitingRun]);-- const enable = () => {- enableTracking.reset();- enableTracking.mutate(- { orgId },- {- onSuccess: () => setAwaitingRun(true)- }- );- };+ const { data: status, isPending } = useGetOrgSecretValueTrackingStatus(orgId, { enabled });return {isLoading: enabled && isPending,// The status route reports completed whenever the org flag is set, so this is the durable answer.- isTrackingOn: status?.status === "completed" && !awaitingRun,+ isTrackingOn: status?.status === "completed",status,- isRunStarting: enableTracking.isPending || (enableTracking.isSuccess && awaitingRun),isEnablePending: enableTracking.isPending,- enable+ enable: () => enableTracking.mutate({ orgId })};};@@ -88,7 +54,7 @@ export const SecretValueTrackingPrompt = ({tracking: TOrgSecretValueTracking;description: string;}) => {- const { status, isLoading, isRunStarting, isEnablePending, enable } = tracking;+ const { status, isLoading, isEnablePending, enable } = tracking;if (isLoading) return <Skeleton className="h-[200px] w-full" />;@@ -106,7 +72,7 @@ export const SecretValueTrackingPrompt = ({);}- if (status?.status === "pending" || isRunStarting) {+ if (status?.status === "pending" || isEnablePending) {return (<div className="flex h-[200px] flex-col items-center justify-center gap-3 text-center"><Loader2Icon className="size-6 animate-spin text-org" />diff --git a/frontend/src/hooks/api/secretInsights/index.ts b/frontend/src/hooks/api/secretInsights/index.tsindex bd43561e88..5cb2a2feff 100644--- a/frontend/src/hooks/api/secretInsights/index.ts+++ b/frontend/src/hooks/api/secretInsights/index.ts@@ -1,9 +1,4 @@-export {- useEnableOrgSecretValueTracking,- useRefreshOrgSecretsDuplication,- useRefreshSecretsDuplication,- useSearchSecretsByValue-} from "./mutations";+export { useEnableOrgSecretValueTracking, useSearchSecretsByValue } from "./mutations";export {secretInsightsKeys,useGetAuthMethodDistribution,diff --git a/frontend/src/hooks/api/secretInsights/mutations.tsx b/frontend/src/hooks/api/secretInsights/mutations.tsxindex cd44f6cc9f..4da1d1dac5 100644--- a/frontend/src/hooks/api/secretInsights/mutations.tsx+++ b/frontend/src/hooks/api/secretInsights/mutations.tsx@@ -2,8 +2,8 @@ import { useMutation, useQueryClient } from "@tanstack/react-query";import { apiRequest } from "@app/config/request";-import { fetchOrgSecretsDuplication, fetchSecretsDuplication, secretInsightsKeys } from "./queries";-import { TSearchSecretsByValueResponse } from "./types";+import { secretInsightsKeys } from "./queries";+import { TGetOrgSecretValueTrackingStatusResponse, TSearchSecretsByValueResponse } from "./types";export const useEnableOrgSecretValueTracking = () => {const queryClient = useQueryClient();@@ -15,6 +15,17 @@ export const useEnableOrgSecretValueTracking = () => {return data;},onSuccess: (_, { orgId }) => {+ // Shown as running straight away, so the previous run's failure does not flash back while+ // the status refetches.+ queryClient.setQueryData<TGetOrgSecretValueTrackingStatusResponse>(+ secretInsightsKeys.orgSecretValueTrackingStatus(orgId),+ (prev) => ({+ status: "pending",+ projectsTotal: prev?.projectsTotal ?? 0,+ projectsDone: 0,+ secretsProcessed: 0+ })+ );queryClient.invalidateQueries({queryKey: secretInsightsKeys.orgSecretValueTrackingStatus(orgId)});@@ -22,30 +33,6 @@ export const useEnableOrgSecretValueTracking = () => {});};-// Recomputes on the server rather than refetching, since a plain refetch answers from the-// server's cache and would report the same stale result.-export const useRefreshSecretsDuplication = () => {- const queryClient = useQueryClient();- return useMutation<unknown, object, { projectId: string }>({- mutationFn: async ({ projectId }) => {- const data = await fetchSecretsDuplication(projectId, true);- queryClient.setQueryData(secretInsightsKeys.secretsDuplication({ projectId }), data);- return data;- }- });-};--export const useRefreshOrgSecretsDuplication = () => {- const queryClient = useQueryClient();- return useMutation<unknown, object, { orgId: string }>({- mutationFn: async ({ orgId }) => {- const data = await fetchOrgSecretsDuplication(true);- queryClient.setQueryData(secretInsightsKeys.orgSecretsDuplication(orgId), data);- return data;- }- });-};-// A mutation rather than a query so the value never becomes part of a cache key.export const useSearchSecretsByValue = () =>useMutation<TSearchSecretsByValueResponse, object, { secretValue: string }>({diff --git a/frontend/src/hooks/api/secretInsights/queries.tsx b/frontend/src/hooks/api/secretInsights/queries.tsxindex 9ffb26f4c7..a852b6b694 100644--- a/frontend/src/hooks/api/secretInsights/queries.tsx+++ b/frontend/src/hooks/api/secretInsights/queries.tsx@@ -198,14 +198,6 @@ export const useGetInsightsCounts = (});};-export const fetchSecretsDuplication = async (projectId: string, refresh = false) => {- const { data } = await apiRequest.get<TGetSecretsDuplicationResponse>(- `/api/v1/insights/${projectId}/secrets/secrets-duplication`,- { params: { refresh } }- );- return data;-};-export const useGetSecretsDuplication = (params: TGetSecretsDuplicationDTO,options?: Omit<@@ -220,7 +212,13 @@ export const useGetSecretsDuplication = () => {return useQuery({queryKey: secretInsightsKeys.secretsDuplication(params),- queryFn: () => fetchSecretsDuplication(params.projectId),+ queryFn: async () => {+ const res = await apiRequest.get<TGetSecretsDuplicationResponse>(+ `/api/v1/insights/${params.projectId}/secrets/secrets-duplication`+ );+ const remainingTtl = Number(res.headers["x-cache-ttl"] ?? -1);+ return { ...res.data, remainingTtl };+ },staleTime: INSIGHTS_STALE_TIME,...options});@@ -335,18 +333,16 @@ export const useGetOrgSecretsAccessVolume = (orgId: string, options?: { enabled?});};-export const fetchOrgSecretsDuplication = async (refresh = false) => {- const { data } = await apiRequest.get<TGetOrgSecretsDuplicationResponse>(- "/api/v1/insights/secrets/secrets-duplication",- { params: { refresh } }- );- return data;-};-export const useGetOrgSecretsDuplication = (orgId: string, options?: { enabled?: boolean }) => {return useQuery({queryKey: secretInsightsKeys.orgSecretsDuplication(orgId),- queryFn: () => fetchOrgSecretsDuplication(),+ queryFn: async () => {+ const res = await apiRequest.get<TGetOrgSecretsDuplicationResponse>(+ "/api/v1/insights/secrets/secrets-duplication"+ );+ const remainingTtl = Number(res.headers["x-cache-ttl"] ?? -1);+ return { ...res.data, remainingTtl };+ },enabled: Boolean(orgId) && (options?.enabled ?? true),staleTime: INSIGHTS_STALE_TIME});diff --git a/frontend/src/hooks/api/secretInsights/types.ts b/frontend/src/hooks/api/secretInsights/types.tsindex 06c563ad7f..0f183f015b 100644--- a/frontend/src/hooks/api/secretInsights/types.ts+++ b/frontend/src/hooks/api/secretInsights/types.ts@@ -143,8 +143,7 @@ export type TDuplicatedSecretGroup = {export type TGetSecretsDuplicationResponse = {secretBlindIndexEnabled: boolean;groups: TDuplicatedSecretGroup[];- // null when detection is not enabled, so nothing was computed- computedAt: string | null;+ remainingTtl: number;};export type TGetSecretBlindIndexStatusDTO = {@@ -232,7 +231,7 @@ export type TOrgDuplicatedSecretGroup = {export type TGetOrgSecretsDuplicationResponse = {orgWideSecretValueTrackingEnabled: boolean;groups: TOrgDuplicatedSecretGroup[];- computedAt: string | null;+ remainingTtl: number;};export type TGetOrgSecretValueTrackingStatusResponse = {diff --git a/frontend/src/pages/organization/SecretInsightsPage/components/DuplicatedSecretsCard.tsx b/frontend/src/pages/organization/SecretInsightsPage/components/DuplicatedSecretsCard.tsxindex ba34ce7a3b..6eaa32a3b1 100644--- a/frontend/src/pages/organization/SecretInsightsPage/components/DuplicatedSecretsCard.tsx+++ b/frontend/src/pages/organization/SecretInsightsPage/components/DuplicatedSecretsCard.tsx@@ -1,12 +1,11 @@-import { useQueryClient } from "@tanstack/react-query";import { AlertTriangleIcon, LockIcon } from "lucide-react";-import { CheckedAtButton } from "@app/components/secrets/CheckedAtButton";import {SecretValueTrackingPrompt,useOrgSecretValueTracking} from "@app/components/secrets/SecretValueTrackingGate";import {+ Badge,Card,CardContent,CardDescription,@@ -21,11 +20,7 @@ import {} from "@app/components/v3";import { OrgPermissionSubjects, useOrganization, useOrgPermission } from "@app/context";import { OrgPermissionSecretsManagementInsightsActions } from "@app/context/OrgPermissionContext/types";-import {- secretInsightsKeys,- useGetOrgSecretsDuplication,- useRefreshOrgSecretsDuplication-} from "@app/hooks/api/secretInsights";+import { useGetOrgSecretsDuplication } from "@app/hooks/api/secretInsights";import { DuplicateGroupList } from "./DuplicateGroupList";@@ -37,7 +32,6 @@ export const DuplicatedSecretsCard = ({ isPlanRestricted }: Props) => {const { currentOrg } = useOrganization();const orgId = currentOrg.id;const { permission } = useOrgPermission();- const queryClient = useQueryClient();const canSearchAllValues = permission.can(OrgPermissionSecretsManagementInsightsActions.SearchAllSecretValues,@@ -45,17 +39,11 @@ export const DuplicatedSecretsCard = ({ isPlanRestricted }: Props) => {);const canQuery = canSearchAllValues && !isPlanRestricted;- const tracking = useOrgSecretValueTracking({- orgId,- enabled: canQuery,- onTrackingOn: () =>- queryClient.invalidateQueries({ queryKey: secretInsightsKeys.orgSecretsDuplication(orgId) })- });+ const tracking = useOrgSecretValueTracking({ orgId, enabled: canQuery });const { data, isPending, isError } = useGetOrgSecretsDuplication(orgId, {enabled: canQuery && tracking.isTrackingOn});- const refresh = useRefreshOrgSecretsDuplication();const renderBody = () => {if (isPlanRestricted) {@@ -129,17 +117,20 @@ export const DuplicatedSecretsCard = ({ isPlanRestricted }: Props) => {return <DuplicateGroupList groups={groups} />;};+ const remainingMinutes =+ canQuery && data?.remainingTtl != null && data.remainingTtl >= 0+ ? Math.max(1, Math.ceil(data.remainingTtl / 60))+ : null;+return (<Card><CardHeader><CardTitle>Duplicated Secrets- {canQuery && data?.computedAt && (- <CheckedAtButton- computedAt={data.computedAt}- isRefreshing={refresh.isPending}- onRefresh={() => refresh.mutate({ orgId })}- />+ {remainingMinutes != null && (+ <Badge variant="neutral" className="ml-2 font-normal">+ Updates in {remainingMinutes} {remainingMinutes === 1 ? "minute" : "minutes"}+ </Badge>)}</CardTitle><CardDescription>diff --git a/frontend/src/pages/secret-manager/InsightsPage/components/DuplicatedSecretsCard.tsx b/frontend/src/pages/secret-manager/InsightsPage/components/DuplicatedSecretsCard.tsxindex 21d7504e2d..50db7a499d 100644--- a/frontend/src/pages/secret-manager/InsightsPage/components/DuplicatedSecretsCard.tsx+++ b/frontend/src/pages/secret-manager/InsightsPage/components/DuplicatedSecretsCard.tsx@@ -11,7 +11,6 @@ import {LockIcon} from "lucide-react";-import { CheckedAtButton } from "@app/components/secrets/CheckedAtButton";import {Accordion,AccordionContent,@@ -47,8 +46,7 @@ import { useEnableSecretBlindIndex } from "@app/hooks/api/projects";import {secretInsightsKeys,useGetSecretBlindIndexStatus,- useGetSecretsDuplication,- useRefreshSecretsDuplication+ useGetSecretsDuplication} from "@app/hooks/api/secretInsights";const overviewRoute =@@ -62,7 +60,6 @@ export const DuplicatedSecretsCard = () => {const [migrationTriggered, setMigrationTriggered] = useState(false);const enableBlindIndex = useEnableSecretBlindIndex();- const refreshDuplicates = useRefreshSecretsDuplication();const { data, isPending } = useGetSecretsDuplication({ projectId }, { enabled: !!projectId });@@ -134,6 +131,11 @@ export const DuplicatedSecretsCard = () => {);};+ const remainingMinutes =+ data?.remainingTtl != null && data.remainingTtl >= 0+ ? Math.max(1, Math.ceil(data.remainingTtl / 60))+ : null;+const isMigrationRunning = migrationTriggered && migrationStatus !== "failed";const isMigrationFailed = migrationStatus === "failed" && !enableBlindIndex.isPending;const showEnableButton = !secretBlindIndexEnabled && !isMigrationRunning && !isMigrationFailed;@@ -143,12 +145,10 @@ export const DuplicatedSecretsCard = () => {<CardHeader><CardTitle>Duplicated Secrets- {data?.computedAt && (- <CheckedAtButton- computedAt={data.computedAt}- isRefreshing={refreshDuplicates.isPending}- onRefresh={() => refreshDuplicates.mutate({ projectId })}- />+ {remainingMinutes != null && (+ <Badge variant="neutral" className="ml-2 font-normal">+ Updates in {remainingMinutes} {remainingMinutes === 1 ? "minute" : "minutes"}+ </Badge>)}</CardTitle><CardDescription>