Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.
TypeScript acmeagentsai-agentscertificate-managementdevsecopsenvironment-variableskubernetesopen-sourcepampkipostgresprivate-caprivileged-access-managementsecret-managementsecret-managersecretssecrets-managementsecuritysecurity-toolsvault
Update secret-read-activity-service.ts
varonix53eac1b922a9
diff --git a/backend/src/ee/services/secret-read-activity/secret-read-activity-service.ts b/backend/src/ee/services/secret-read-activity/secret-read-activity-service.tsindex ad8204c1fc..ee04b9dfde 100644--- a/backend/src/ee/services/secret-read-activity/secret-read-activity-service.ts+++ b/backend/src/ee/services/secret-read-activity/secret-read-activity-service.ts@@ -1,4 +1,6 @@-import { ProjectMembershipRole, SecretType, TableName } from "@app/db/schemas";+import { setImmediate } from "node:timers/promises";++import { SecretType, TableName } from "@app/db/schemas";import { DAY_IN_MS } from "@app/ee/services/audit-report/audit-report-fns";import { TIdentityGroupMembershipDALFactory } from "@app/ee/services/group/identity-group-membership-dal";import { TUserGroupMembershipDALFactory } from "@app/ee/services/group/user-group-membership-dal";@@ -21,7 +23,6 @@ import { TKmsServiceFactory } from "@app/services/kms/kms-service";import { KmsDataKey } from "@app/services/kms/kms-types";import { TProjectDALFactory } from "@app/services/project/project-dal";import { TProjectEnvDALFactory } from "@app/services/project-env/project-env-dal";-import { TRoleDALFactory } from "@app/services/role/role-dal";import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";import { TSecretImportDALFactory } from "@app/services/secret-import/secret-import-dal";import { getAllSecretReferences } from "@app/services/secret-v2-bridge/secret-reference-fns";@@ -32,28 +33,32 @@ import { TSecretReadActivityDALFactory } from "./secret-read-activity-dal";import {classifySecretReads,computeSecretReach,- coversWholeWindow,getReadActivityWindowStart,indexSecretReads,loadProjectFolderTree,SECRET_READ_ACTIVITY_WINDOW_DAYS,+ summarizeSecretAccess,+ toActorKey,toLocationKey,- TProjectFolderTree+ toUnusedAccessRoles} from "./secret-read-activity-fns";import { TSecretReadActivityQueueServiceFactory } from "./secret-read-activity-queue";import {TEntitledReader,TGetSecretReadActivityDTO,- TGetUnusedAccessPrincipalDTO,+ TGetUnusedAccessActorDTO,TListUnusedAccessDTO,+ TProjectReader,TReachSecret,TReadActivityScope,+ TSecretActivity,TSecretReach,TSecretReachGraph,TSecretReaderType,- TUnusedAccessPrincipal,- TUnusedSecret,- UnusedAccessPrincipalType+ TUnusedAccessActor,+ TUnusedAccessSummary,+ TUnusedAccessTally,+ TUnusedSecret} from "./secret-read-activity-types";const UNUSED_ACCESS_CACHE_TTL_SECONDS = 24 * 60 * 60;@@ -61,17 +66,10 @@ const REACH_YIELD_INTERVAL = 100;// Longer than a computation can take, so the lock is not lost mid-scan; waiters give up well before a request times out.const UNUSED_ACCESS_COMPUTE_LOCK_TTL_MS = 5 * 60 * 1000;const UNUSED_ACCESS_COMPUTE_WAIT_SECONDS = 60;-// Every principal's row is cached, but the unread secret ids behind the rows only for the principals with the most-// of them, capped per principal, which keeps a project's entry under about 1.5 MB.-const UNUSED_ACCESS_DETAIL_MAX_PRINCIPALS = 200;-const UNUSED_ACCESS_MAX_SECRETS_PER_PRINCIPAL = 200;-const ROLE_LABELS: Record<string, string> = {- admin: "Admin",- member: "Member",- viewer: "Viewer",- "no-access": "No Access"-};-type TSecretPermission = Parameters<typeof hasSecretReadValueOrDescribePermission>[0];+// Every actor's row is cached, but the unread secret ids behind the rows only for the actors with the most of them,+// capped per actor, which keeps a project's entry under about 1.5 MB.+const UNUSED_ACCESS_DETAIL_MAX_ACTORS = 200;+const UNUSED_ACCESS_MAX_SECRETS_PER_ACTOR = 200;type TSecretReadActivityServiceFactoryDep = {permissionService: Pick<TPermissionServiceFactory, "getProjectPermission" | "getProjectPermissions">;@@ -89,7 +87,6 @@ type TSecretReadActivityServiceFactoryDep = {identityDAL: Pick<TIdentityDALFactory, "find">;userGroupMembershipDAL: Pick<TUserGroupMembershipDALFactory, "find">;identityGroupMembershipDAL: Pick<TIdentityGroupMembershipDALFactory, "find">;- roleDAL: Pick<TRoleDALFactory, "find">;kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;keyStore: Pick<TKeyStoreFactory, "setItemWithExpiry" | "getItem" | "ttl" | "acquireLock">;};@@ -109,7 +106,6 @@ export const secretReadActivityServiceFactory = ({identityDAL,userGroupMembershipDAL,identityGroupMembershipDAL,- roleDAL,kmsService,keyStore}: TSecretReadActivityServiceFactoryDep) => {@@ -127,13 +123,13 @@ export const secretReadActivityServiceFactory = ({windowStart,secret: requestedSecret,tree: loadedTree- }: TReadActivityScope & { tree?: TProjectFolderTree }) => {+ }: TReadActivityScope): Promise<TSecretActivity[]> => {const tree = loadedTree ?? (await loadProjectFolderTree({ folderDAL, projectEnvDAL }, projectId));- const { folders, folderById, locationOf, pathOf } = tree;+ const { folders, folderById, folderIdByLocation, locationOf, pathOf } = tree;const requestedFolderId =requestedSecret &&- tree.folderIdByLocation.get(+ folderIdByLocation.get(toLocationKey({ environment: requestedSecret.environment, secretPath: requestedSecret.secretPath }));if (requestedSecret && !requestedFolderId) {@@ -145,19 +141,16 @@ export const secretReadActivityServiceFactory = ({? [requestedFolderId]: folders.filter((folder) => !folder.isReserved).map((folder) => folder.id);- const referencesByLocation = groupBy(- await secretV2BridgeDAL.findSecretReferencesByProjectId(projectId),- (reference) => toLocationKey(reference)- );- const referencesOf = (folderId: string) => {- const location = locationOf(folderId);- return location ? (referencesByLocation[toLocationKey(location)] ?? []) : [];- };+ // Each reference row is a secret (secretId, folderId) that references a key in another folder.+ const references = (await secretV2BridgeDAL.findSecretReferencesByProjectId(projectId)).flatMap((reference) => {+ const targetFolderId = folderIdByLocation.get(toLocationKey(reference));+ return targetFolderId ? [{ ...reference, targetFolderId }] : [];+ });+ const referencesByTargetFolderId = groupBy(references, (reference) => reference.targetFolderId);// Secrets referencing a loaded one can sit outside the scope, so their folders are loaded too, until no// new folders turn up.- type TSecret = Awaited<ReturnType<typeof secretV2BridgeDAL.find>>[number];- const secretsByFolder = new Map<string, TSecret[]>();+ const secretsByFolder = new Map<string, TSecretActivity["secret"][]>();let pendingFolderIds = scopeFolderIds;while (pendingFolderIds.length) {const folderIds = pendingFolderIds;@@ -167,43 +160,36 @@ export const secretReadActivityServiceFactory = ({});const secretsByFolderId = groupBy(secrets, (secret) => secret.folderId);folderIds.forEach((folderId) => secretsByFolder.set(folderId, secretsByFolderId[folderId] ?? []));- pendingFolderIds = [...new Set(folderIds.flatMap(referencesOf).map((reference) => reference.folderId))].filter(+ const referencingFolderIds = folderIds.flatMap((folderId) =>+ (referencesByTargetFolderId[folderId] ?? []).map((reference) => reference.folderId)+ );+ pendingFolderIds = [...new Set(referencingFolderIds)].filter((folderId) => folderById[folderId] && !secretsByFolder.has(folderId));}+ // Every loaded secret is decrypted once to find the keys it references in its own folder. The edges are indexed+ // by "<referenced folder>:<referenced key>", since scanning a folder for every secret in it would be quadratic.const { decryptor } = await kmsService.createCipherPairWithDataKey({ type: KmsDataKey.SecretManager, projectId });- const secretById = new Map([...secretsByFolder.values()].flat().map((secret) => [secret.id, secret]));-- // Indexed by referenced key once per folder: scanning the folder again for every secret in it would be- // quadratic in the folder's size, and the reach is computed synchronously.- const referencersByFolder = new Map<string, Map<string, TReachSecret[]>>();- const referencersOf = (folderId: string) => {- const indexed = referencersByFolder.get(folderId);- if (indexed) return indexed;-- const referencersByKey = new Map<string, TReachSecret[]>();- const add = (key: string, referencer: TReachSecret) => {- const referencers = referencersByKey.get(key);- if (referencers) referencers.push(referencer);- else referencersByKey.set(key, [referencer]);- };- (secretsByFolder.get(folderId) ?? []).forEach((secret) => {- if (!secret.encryptedValue) return;+ const loadedSecrets = [...secretsByFolder.values()].flat();+ const secretById = new Map(loadedSecrets.map((secret) => [secret.id, secret]));+ const referenceEdges: { target: string; referencer: TReachSecret }[] = [+ ...loadedSecrets.flatMap((secret) => {+ if (!secret.encryptedValue) return [];const { localReferences } = getAllSecretReferences(decryptor({ cipherTextBlob: secret.encryptedValue }).toString());- new Set(localReferences).forEach((key) => add(key, secret));- });- referencesOf(folderId).forEach((reference) =>- add(- reference.secretKey,- secretById.get(reference.secretId) ?? { id: reference.secretId, folderId: reference.folderId }- )- );- referencersByFolder.set(folderId, referencersByKey);- return referencersByKey;- };+ return [...new Set(localReferences)].map((key) => ({+ target: `${secret.folderId}:${key}`,+ referencer: secret+ }));+ }),+ ...references.map((reference) => ({+ target: `${reference.targetFolderId}:${reference.secretKey}`,+ referencer: secretById.get(reference.secretId) ?? { id: reference.secretId, folderId: reference.folderId }+ }))+ ];+ const referenceEdgesByTarget = groupBy(referenceEdges, (edge) => edge.target);// Replication imports are included: a replicated copy is checked against the reader's access to its source.const importsByLocation = groupBy(await secretImportDAL.find({ projectId }), (secretImport) =>@@ -211,7 +197,11 @@ export const secretReadActivityServiceFactory = ({);const graph: TSecretReachGraph = {getReferencers: ({ id, folderId, key }) =>- key === undefined ? [] : (referencersOf(folderId).get(key) ?? []).filter((referencer) => referencer.id !== id),+ key === undefined+ ? []+ : (referenceEdgesByTarget[`${folderId}:${key}`] ?? []).flatMap(({ referencer }) =>+ referencer.id === id ? [] : [referencer]+ ),getImporterFolderIds: (folderId) =>folderById[folderId]? (@@ -232,14 +222,13 @@ export const secretReadActivityServiceFactory = ({).map((secret) => ({ secret, ...location }));});- // Each reach is synchronous work, so yield between batches to keep a large project from blocking other requests.+ // Each reach is synchronous work, so the loop yields to the event loop between batches to keep a large project+ // from blocking other requests.const reaches: TSecretReach[] = [];for (let index = 0; index < targets.length; index += 1) {if (index > 0 && index % REACH_YIELD_INTERVAL === 0) {// eslint-disable-next-line no-await-in-loop- await new Promise<void>((resolve) => {- setImmediate(resolve);- });+ await setImmediate();}reaches.push(computeSecretReach(targets[index].secret, graph));}@@ -271,7 +260,6 @@ export const secretReadActivityServiceFactory = ({};});};- type TSecretActivity = Awaited<ReturnType<typeof $getReadActivity>>[number];// Reads are known from the later of when tracking began on the instance and when the project was created.const $getProjectTrackingStartedAt = async (projectId: string) => {@@ -282,11 +270,11 @@ export const secretReadActivityServiceFactory = ({};};- // How each secret stands, from everyone who can read its value today: unused access is judged on the- // non-admins among them, since admins can read everything by design; ghost readers are readers in the window- // who are not among them at all, so their access was since removed. Groups are judged on their own too: a group's access to a secret is unused when none of its- // members read it. Until a project has a full window of reads, it is judged on the reads since tracking began.- const $loadSecretSummarizer = async (projectId: string, orgId: string, windowStart: Date) => {+ // Everyone who can hold secrets in the project, and what each secret's readers are judged against. The user and+ // identity lists carry only direct members, so every group member is also listed with what the group gives them,+ // their own folder grants included. Until a project has a full window of reads, it is judged on the reads since+ // tracking began.+ const $loadProjectAccess = async (projectId: string, orgId: string, windowStart: Date) => {const [{ userPermissions, identityPermissions, groupPermissions },accessChangedAtByActorId,@@ -298,115 +286,70 @@ export const secretReadActivityServiceFactory = ({secretReadActivityDAL.findAdminActorIds(projectId),$getProjectTrackingStartedAt(projectId)]);- // The user and identity lists carry only direct members, so every group member is also checked with what the- // group gives them, their own folder grants included; an actor who qualifies more than one way counts once.const groupIds = groupPermissions.map((group) => group.id);const [userGroupMemberships, identityGroupMemberships] = await Promise.all([groupIds.length ? userGroupMembershipDAL.find({ $in: { groupId: groupIds } }) : [],groupIds.length ? identityGroupMembershipDAL.find({ $in: { groupId: groupIds } }) : []]);- const groupPermissionById = new Map(groupPermissions.map((group) => [group.id, group]));- // In a project tracked for less than the window, unused access is judged on the reads since tracking began.- const judgedFrom = trackingStartedAt > windowStart ? trackingStartedAt : windowStart;- type TActor = {- id: string;- name?: string;- actorType: TSecretReaderType;- permission: (typeof userPermissions)[number]["permission"];- };- const throughGroup = <T extends { groupId: string }>(- memberships: T[],- actorType: TSecretReaderType,- idOf: (membership: T) => string- ): TActor[] =>- memberships.flatMap((membership) => {- const group = groupPermissionById.get(membership.groupId);- const id = idOf(membership);- return group ? [{ id, actorType, permission: group.permissionForMember(id) }] : [];- });- const actors: TActor[] = [- ...userPermissions.map((actor) => ({ ...actor, actorType: ActorType.USER as const })),- ...identityPermissions.map((actor) => ({ ...actor, actorType: ActorType.IDENTITY as const })),- ...throughGroup(userGroupMemberships, ActorType.USER, (membership) => membership.userId),- ...throughGroup(identityGroupMemberships, ActorType.IDENTITY, (membership) => membership.identityId)- ];- // Actors with identical rules always get the same answer, so each distinct rule set is checked once.- const actorGroups = Object.values(groupBy(actors, (actor) => JSON.stringify(actor.permission.rules)));- const actorNames = new Map<string, string>(- actors.flatMap((actor) => (actor.name ? [[`${actor.actorType}:${actor.id}`, actor.name] as const] : []))- );- // Admin groups can read everything by design, so like admins they are left out.- const memberKeysByGroupId = new Map<string, string[]>();- const addMember = (groupId: string, key: string) =>- memberKeysByGroupId.set(groupId, [...(memberKeysByGroupId.get(groupId) ?? []), key]);- userGroupMemberships.forEach((membership) =>- addMember(membership.groupId, `${ActorType.USER}:${membership.userId}`)- );- identityGroupMemberships.forEach((membership) =>- addMember(membership.groupId, `${ActorType.IDENTITY}:${membership.identityId}`)- );- const groups = groupPermissions- .filter((group) => !group.roles.some((role) => role.role === ProjectMembershipRole.Admin))- .map((group) => ({ ...group, memberKeys: memberKeysByGroupId.get(group.id) ?? [] }));+ const groupById = new Map(groupPermissions.map((group) => [group.id, group]));+ const groupMembers = [+ ...userGroupMemberships.map(({ groupId, userId }) => ({ groupId, actorType: ActorType.USER, actorId: userId })),+ ...identityGroupMemberships.map(({ groupId, identityId }) => ({+ groupId,+ actorType: ActorType.IDENTITY,+ actorId: identityId+ }))+ ].flatMap(({ groupId, actorType, actorId }) => {+ const group = groupById.get(groupId);+ return group ? [{ group, actorType: actorType as TSecretReaderType, actorId }] : [];+ });+ const readers: TProjectReader[] = [+ ...userPermissions.map(({ id, permission }) => ({ actorType: ActorType.USER as const, actorId: id, permission })),+ ...identityPermissions.map(({ id, permission }) => ({+ actorType: ActorType.IDENTITY as const,+ actorId: id,+ permission+ })),+ ...groupMembers.map(({ group, actorType, actorId }) => ({+ actorType,+ actorId,+ permission: group.permissionForMember(actorId)+ }))+ ];- const summarize = ({ secret, environment, secretPath, trackedSince, reads }: TSecretActivity) => {- const subject = {- environment,- secretPath,- secretName: secret.key,- secretTags: secret.tags?.map((tag) => tag.slug)- };- const canReadValue = (permission: TActor["permission"]) =>- hasSecretReadValueOrDescribePermission(permission, ProjectPermissionSecretActions.ReadValue, subject);- const entitled = [- ...new Map(- actorGroups- .filter(([{ permission }]) => canReadValue(permission))- .flat()- .map((actor) => [`${actor.actorType}:${actor.id}`, actor] as const)- ).values()- ];- const entitledKeys = new Set(entitled.map((actor) => `${actor.actorType}:${actor.id}`));- // Admins can read everything by design, so they are left out of the counts, though their reads still show,- // and a read of theirs is never a ghost's.- const nonAdmins = entitled.filter((actor) => !adminActorIds.has(actor.id));- const unread = nonAdmins.filter(- (actor) =>- !reads.has(`${actor.actorType}:${actor.id}`) &&- coversWholeWindow(trackedSince, accessChangedAtByActorId.get(actor.id), judgedFrom)- );- const ghostReaders = [...reads.values()].filter(- (read) => !entitledKeys.has(`${read.actorType}:${read.actorId}`) && !adminActorIds.has(read.actorId)- );- const entitledGroups = groups.filter((group) => canReadValue(group.permission));- const unreadGroups = entitledGroups.filter(- (group) =>- !group.memberKeys.some((key) => reads.has(key)) &&- coversWholeWindow(trackedSince, accessChangedAtByActorId.get(group.id), judgedFrom)- );- return {- entitled,- nonAdmins,- unread,- ghostReaders,- entitledGroups,- unreadGroups- };- };return {- summarize,- actorNames,- groups,- userPermissions,- identityPermissions,- judgedFrom,+ readersByRules: Object.values(groupBy(readers, (reader) => JSON.stringify(reader.permission.rules))),+ adminActorIds,accessChangedAtByActorId,- adminActorIds+ judgedFrom: trackingStartedAt > windowStart ? trackingStartedAt : windowStart,+ directMembers: new Map([+ ...userPermissions.map((member) => [toActorKey(ActorType.USER, member.id), member] as const),+ ...identityPermissions.map((member) => [toActorKey(ActorType.IDENTITY, member.id), member] as const)+ ]),+ groupMembershipsByActorKey: groupBy(groupMembers, (member) => toActorKey(member.actorType, member.actorId))};};- type TSummarizer = Awaited<ReturnType<typeof $loadSecretSummarizer>>;- type TSecretSummary = ReturnType<TSummarizer["summarize"]>;++ // Direct members come named. Everyone else (members only through a group, and readers since gone from the+ // project) is looked up; a user or identity deleted outright has no name.+ const $findActorNames = async (+ actors: { actorType: string; actorId: string }[],+ directMembers: Map<string, { name: string }>+ ) => {+ const others = actors.filter((actor) => !directMembers.has(toActorKey(actor.actorType, actor.actorId)));+ const userIds = [...new Set(others.filter((a) => a.actorType === ActorType.USER).map((a) => a.actorId))];+ const identityIds = [...new Set(others.filter((a) => a.actorType === ActorType.IDENTITY).map((a) => a.actorId))];+ const [users, identities] = await Promise.all([+ userIds.length ? userDAL.find({ $in: { id: userIds } }) : [],+ identityIds.length ? identityDAL.find({ $in: { id: identityIds } }) : []+ ]);+ return new Map([+ ...[...directMembers].map(([key, member]) => [key, member.name] as const),+ ...users.map((user) => [toActorKey(ActorType.USER, user.id), user.username] as const),+ ...identities.map((identity) => [toActorKey(ActorType.IDENTITY, identity.id), identity.name] as const)+ ]);+ };const getSecretReadActivity = async ({ projectId, environment, secretPath, secretName }: TGetSecretReadActivityDTO,@@ -437,266 +380,130 @@ export const secretReadActivityServiceFactory = ({});const windowStart = getReadActivityWindowStart();- const [[activity], { summarize, actorNames, judgedFrom, accessChangedAtByActorId, adminActorIds }] =- await Promise.all([- $getReadActivity({ projectId, windowStart, secret: { environment, secretPath, secretName }, tree }),- $loadSecretSummarizer(projectId, actor.orgId, windowStart)- ]);+ const [[activity], access] = await Promise.all([+ $getReadActivity({ projectId, windowStart, secret: { environment, secretPath, secretName }, tree }),+ $loadProjectAccess(projectId, actor.orgId, windowStart)+ ]);if (!activity) {throw new NotFoundError({message: `Secret with name '${secretName}' not found at path '${secretPath}' in environment '${environment}'`});}- const { entitled, nonAdmins, unread, ghostReaders } = summarize(activity);- const unreadKeys = new Set(unread.map((reader) => `${reader.actorType}:${reader.id}`));-- // A ghost reader may have left the project or been deleted since, and a reader who has the value through a- // group is not among the direct members, so names the members do not give are looked up directly; a reader- // deleted outright has none.- const unnamed = [- ...ghostReaders,- ...entitled.map((reader) => ({ actorType: reader.actorType, actorId: reader.id }))- ].filter((read) => !actorNames.has(`${read.actorType}:${read.actorId}`));- const idsOf = (actorType: TSecretReaderType) => [- ...new Set(unnamed.filter((read) => read.actorType === actorType).map((read) => read.actorId))- ];- const [userIds, identityIds] = [idsOf(ActorType.USER), idsOf(ActorType.IDENTITY)];- const [users, identities] = await Promise.all([- userIds.length ? userDAL.find({ $in: { id: userIds } }) : [],- identityIds.length ? identityDAL.find({ $in: { id: identityIds } }) : []- ]);- users.forEach((user) => actorNames.set(`${ActorType.USER}:${user.id}`, user.username));- identities.forEach((identity) => actorNames.set(`${ActorType.IDENTITY}:${identity.id}`, identity.name));+ const { entitled, nonAdmins, unread, ghostReaders } = summarizeSecretAccess(activity, access);+ const unreadKeys = new Set(unread.map((reader) => toActorKey(reader.actorType, reader.actorId)));+ const names = await $findActorNames([...entitled, ...ghostReaders], access.directMembers);return {trackedSince: activity.trackedSince,- judgedFrom,+ judgedFrom: access.judgedFrom,reads: [...activity.reads.values()],- entitled: entitled.map(- (reader): TEntitledReader => ({+ entitled: entitled.map((reader): TEntitledReader => {+ const key = toActorKey(reader.actorType, reader.actorId);+ return {actorType: reader.actorType,- actorId: reader.id,- actorName: actorNames.get(`${reader.actorType}:${reader.id}`) ?? null,- unread: unreadKeys.has(`${reader.actorType}:${reader.id}`),- isAdmin: adminActorIds.has(reader.id),- accessChangedAt: accessChangedAtByActorId.get(reader.id) ?? null- })- ),+ actorId: reader.actorId,+ actorName: names.get(key) ?? null,+ unread: unreadKeys.has(key),+ isAdmin: access.adminActorIds.has(reader.actorId),+ accessChangedAt: access.accessChangedAtByActorId.get(reader.actorId) ?? null+ };+ }),entitledCount: nonAdmins.length,unreadCount: unread.length,ghostReaderCount: ghostReaders.length,ghostReaders: ghostReaders.map((read) => ({...read,- actorName: actorNames.get(`${read.actorType}:${read.actorId}`) ?? null+ actorName: names.get(toActorKey(read.actorType, read.actorId)) ?? null}))};};- // Walks every secret once and hands each one's summary to `visit`. The permission checks are synchronous, so it- // yields between secrets to keep a large project from blocking other requests.- const $walkSecrets = async (- projectId: string,- orgId: string,- visit: (activity: TSecretActivity, summary: TSecretSummary) => void- ) => {+ // Scans every secret once and tallies, per user and machine identity, the secrets it can read and the ones it has+ // not read in the window. The permission checks are synchronous, so the scan yields to the event loop between+ // secrets to keep a large project from blocking other requests.+ const $tallyUnusedAccess = async (projectId: string, orgId: string) => {const windowStart = getReadActivityWindowStart();- const [activities, summarizer] = await Promise.all([+ const [activities, access] = await Promise.all([$getReadActivity({ projectId, windowStart }),- $loadSecretSummarizer(projectId, orgId, windowStart)+ $loadProjectAccess(projectId, orgId, windowStart)]);- for (let index = 0; index < activities.length; index += 1) {+ const tallies = new Map<string, TUnusedAccessTally>();+ const lastReadAtByKey = new Map<string, Date>();+ for (const activity of activities) {// eslint-disable-next-line no-await-in-loop- await new Promise<void>((resolve) => {- setImmediate(resolve);- });- visit(activities[index], summarizer.summarize(activities[index]));- }- return summarizer;- };-- const toPrincipalType = (actorType: TSecretReaderType) =>- actorType === ActorType.USER ? UnusedAccessPrincipalType.User : UnusedAccessPrincipalType.Identity;- const principalKey = (principalType: UnusedAccessPrincipalType, principalId: string) =>- `${principalType}:${principalId}`;-- // Everyone who can read secrets they have not read in the window: users and machine identities, whether through- // a role, a privilege or a group, and the groups themselves. Ranked by how many such secrets they have.- const $computeUnusedAccess = async (projectId: string, orgId: string) => {- type TStat = {- principalType: UnusedAccessPrincipalType;- principalId: string;- entitledCount: number;- unreadCount: number;- unreadSecretIds: string[];- environments: Set<string>;- };- const stats = new Map<string, TStat>();- const statOf = (principalType: UnusedAccessPrincipalType, principalId: string) => {- const key = principalKey(principalType, principalId);- let stat = stats.get(key);- if (!stat) {- stat = {- principalType,- principalId,+ await setImmediate();+ const { nonAdmins, unread } = summarizeSecretAccess(activity, access);+ nonAdmins.forEach(({ actorType, actorId }) => {+ const key = toActorKey(actorType, actorId);+ const tally = tallies.get(key) ?? {+ actorType,+ actorId,entitledCount: 0,unreadCount: 0,unreadSecretIds: [],environments: new Set()};- stats.set(key, stat);- }- return stat;- };- const markUnread = (principalType: UnusedAccessPrincipalType, principalId: string, activity: TSecretActivity) => {- const stat = statOf(principalType, principalId);- stat.unreadCount += 1;- stat.environments.add(activity.environment);- if (stat.unreadSecretIds.length < UNUSED_ACCESS_MAX_SECRETS_PER_PRINCIPAL) {- stat.unreadSecretIds.push(activity.secret.id);- }- };- const lastReadAtByKey = new Map<string, number>();-- const { actorNames, groups, userPermissions, identityPermissions } = await $walkSecrets(- projectId,- orgId,- (activity, summary) => {- summary.nonAdmins.forEach((actor) => {- statOf(toPrincipalType(actor.actorType), actor.id).entitledCount += 1;- });- summary.unread.forEach((actor) => markUnread(toPrincipalType(actor.actorType), actor.id, activity));- summary.entitledGroups.forEach((group) => {- statOf(UnusedAccessPrincipalType.Group, group.id).entitledCount += 1;- });- summary.unreadGroups.forEach((group) => markUnread(UnusedAccessPrincipalType.Group, group.id, activity));- activity.reads.forEach((read, key) => {- lastReadAtByKey.set(key, Math.max(lastReadAtByKey.get(key) ?? 0, read.lastReadAt.getTime()));- });- }- );+ tally.entitledCount += 1;+ tallies.set(key, tally);+ });+ // Every unread reader is also a non-admin, so its tally exists.+ unread.forEach(({ actorType, actorId }) => {+ const tally = tallies.get(toActorKey(actorType, actorId)) as TUnusedAccessTally;+ tally.unreadCount += 1;+ tally.environments.add(activity.environment);+ if (tally.unreadSecretIds.length < UNUSED_ACCESS_MAX_SECRETS_PER_ACTOR) {+ tally.unreadSecretIds.push(activity.secret.id);+ }+ });+ activity.reads.forEach((read, key) => {+ const latest = lastReadAtByKey.get(key);+ if (!latest || read.lastReadAt > latest) lastReadAtByKey.set(key, read.lastReadAt);+ });+ }+ return { tallies, lastReadAtByKey, access };+ };- const ranked = [...stats.values()]- .filter((stat) => stat.unreadCount > 0)+ // Every user and machine identity that can read secrets it has not read in the window, whether through a role, a+ // privilege or a group. Ranked by how many such secrets it has.+ const $computeUnusedAccess = async (projectId: string, orgId: string): Promise<TUnusedAccessSummary> => {+ const { tallies, lastReadAtByKey, access } = await $tallyUnusedAccess(projectId, orgId);+ const ranked = [...tallies.values()]+ .filter((tally) => tally.unreadCount >= 0).sort((a, b) => b.unreadCount - a.unreadCount || a.entitledCount - b.entitledCount);+ const names = await $findActorNames(ranked, access.directMembers);- // Direct members and groups come named; members who have the access only through a group are looked up.- groups.forEach((group) => actorNames.set(principalKey(UnusedAccessPrincipalType.Group, group.id), group.name));- const unnamed = ranked.filter((stat) => !actorNames.has(principalKey(stat.principalType, stat.principalId)));- const idsOf = (principalType: UnusedAccessPrincipalType) =>- unnamed.filter((stat) => stat.principalType === principalType).map((stat) => stat.principalId);- const [userIds, identityIds] = [idsOf(UnusedAccessPrincipalType.User), idsOf(UnusedAccessPrincipalType.Identity)];- const [users, identities] = await Promise.all([- userIds.length ? userDAL.find({ $in: { id: userIds } }) : [],- identityIds.length ? identityDAL.find({ $in: { id: identityIds } }) : []- ]);- users.forEach((user) => actorNames.set(principalKey(UnusedAccessPrincipalType.User, user.id), user.username));- identities.forEach((identity) =>- actorNames.set(principalKey(UnusedAccessPrincipalType.Identity, identity.id), identity.name)- );-- // How each principal gets the access, from the roles its permission was built from. Custom roles are shown by- // name, which is the one lookup the computation makes beyond what the permissions already loaded.- const rolesByKey = new Map<string, { role: string; customRoleSlug?: string | null }[]>();- const privilegedKeys = new Set<string>();- userPermissions.forEach((entry) => {- rolesByKey.set(principalKey(UnusedAccessPrincipalType.User, entry.id), entry.roles);- if (entry.hasAdditionalPrivileges) privilegedKeys.add(principalKey(UnusedAccessPrincipalType.User, entry.id));- });- identityPermissions.forEach((entry) => {- rolesByKey.set(principalKey(UnusedAccessPrincipalType.Identity, entry.id), entry.roles);- if (entry.hasAdditionalPrivileges) privilegedKeys.add(principalKey(UnusedAccessPrincipalType.Identity, entry.id));- });- groups.forEach((group) => rolesByKey.set(principalKey(UnusedAccessPrincipalType.Group, group.id), group.roles));- const customRoleSlugs = [- ...new Set([...rolesByKey.values()].flat().flatMap((role) => (role.customRoleSlug ? [role.customRoleSlug] : [])))- ];- const customRoles = customRoleSlugs.length ? await roleDAL.find({ projectId, $in: { slug: customRoleSlugs } }) : [];- const customRoleNameBySlug = new Map(customRoles.map((role) => [role.slug, role.name]));- const groupNamesByMemberKey = new Map<string, string[]>();- groups.forEach((group) =>- group.memberKeys.forEach((key) =>- groupNamesByMemberKey.set(key, [...(groupNamesByMemberKey.get(key) ?? []), group.name])- )- );- const roleLabel = (role: { role: string; customRoleSlug?: string | null }) =>- role.role === ProjectMembershipRole.Custom && role.customRoleSlug- ? `Role: ${customRoleNameBySlug.get(role.customRoleSlug) ?? role.customRoleSlug}`- : `Role: ${ROLE_LABELS[role.role] ?? role.role}`;- const accessViaOf = (key: string) => [- ...new Set([- ...(rolesByKey.get(key) ?? []).map(roleLabel),- ...(privilegedKeys.has(key) ? ["Additional privilege"] : []),- ...(groupNamesByMemberKey.get(key) ?? []).map((name) => `Group: ${name}`)- ])- ];- // A group's last read is its members' latest.- const lastReadOf = (stat: TStat) => {- const keys =- stat.principalType === UnusedAccessPrincipalType.Group- ? (groups.find((group) => group.id === stat.principalId)?.memberKeys ?? [])- : [principalKey(stat.principalType, stat.principalId)];- const latest = Math.max(0, ...keys.map((key) => lastReadAtByKey.get(key) ?? 0));- return latest ? new Date(latest) : null;- };-- const rows = ranked.map((stat): TUnusedAccessPrincipal => {- const key = principalKey(stat.principalType, stat.principalId);+ const rows = ranked.map((tally): TUnusedAccessActor => {+ const key = toActorKey(tally.actorType, tally.actorId);+ const member = access.directMembers.get(key);return {- principalType: stat.principalType,- principalId: stat.principalId,- name: actorNames.get(key) ?? stat.principalId,- memberCount:- stat.principalType === UnusedAccessPrincipalType.Group- ? (groups.find((group) => group.id === stat.principalId)?.memberKeys.length ?? 0)- : null,- entitledCount: stat.entitledCount,- unreadCount: stat.unreadCount,- environments: [...stat.environments].sort(),- lastReadAt: lastReadOf(stat),- accessVia: accessViaOf(key)+ actorType: tally.actorType,+ actorId: tally.actorId,+ name: names.get(key) ?? tally.actorId,+ entitledCount: tally.entitledCount,+ unreadCount: tally.unreadCount,+ environments: [...tally.environments].sort(),+ lastReadAt: lastReadAtByKey.get(key) ?? null,+ accessPaths: {+ roles: toUnusedAccessRoles(member?.roles ?? []),+ groups: (access.groupMembershipsByActorKey[key] ?? []).map(({ group }) => ({+ id: group.id,+ name: group.name,+ roles: toUnusedAccessRoles(group.roles)+ })),+ additionalPrivileges: member?.additionalPrivileges ?? []+ }};});- // The unread secrets themselves are kept for the principals with the most of them, so the sheet lists them- // without another scan; anyone further down is computed on their own when opened.+ // The unread secrets themselves are kept for the actors with the most of them, so the sheet lists them without+ // another scan; anyone further down is scanned for on their own when opened.const details = Object.fromEntries(ranked- .slice(0, UNUSED_ACCESS_DETAIL_MAX_PRINCIPALS)- .map((stat) => [principalKey(stat.principalType, stat.principalId), stat.unreadSecretIds])+ .slice(0, UNUSED_ACCESS_DETAIL_MAX_ACTORS)+ .map((tally) => [toActorKey(tally.actorType, tally.actorId), tally.unreadSecretIds]));return { rows, details };};- // Requests on a cold cache, on any instance, wait for the one computing the project and read what it cached- // instead of each scanning the project. If the wait runs out, the request computes it itself.- const $computeUnusedAccessShared = async (projectId: string, orgId: string, cacheKey: string) => {- let lock: Awaited<ReturnType<typeof keyStore.acquireLock>> | undefined;- try {- lock = await keyStore.acquireLock(- [KeyStorePrefixes.SecretReadActivityUnusedAccessLock(projectId)],- UNUSED_ACCESS_COMPUTE_LOCK_TTL_MS,- { retryCount: UNUSED_ACCESS_COMPUTE_WAIT_SECONDS, retryDelay: 1000 }- );- } catch {- lock = undefined;- }- try {- const cached = lock ? await keyStore.getItem(cacheKey) : null;- if (cached) return JSON.parse(cached) as Awaited<ReturnType<typeof $computeUnusedAccess>>;-- const result = await $computeUnusedAccess(projectId, orgId);- // Written before the lock is released, so the next request in line finds it. A failed write only costs- // the next request a recomputation, so it never fails this one.- await keyStore- .setItemWithExpiry(cacheKey, UNUSED_ACCESS_CACHE_TTL_SECONDS, JSON.stringify(result))- .catch((error) => {- logger.warn(error, `unused-access: failed to cache the project computation [projectId=${projectId}]`);- });- return result;- } finally {- await lock?.release();- }- };-// The whole project is computed once a day and cached, so paging through it is cheap.const $getCachedUnusedAccess = async (projectId: string, orgId: string) => {const cacheKey = KeyStorePrefixes.InsightsCache(projectId, "unused-access");@@ -704,7 +511,36 @@ export const secretReadActivityServiceFactory = ({keyStore,key: cacheKey,ttlSeconds: UNUSED_ACCESS_CACHE_TTL_SECONDS,- fetcher: () => $computeUnusedAccessShared(projectId, orgId, cacheKey)+ // Requests on a cold cache, on any instance, wait for the one computing the project and read what it cached+ // instead of each scanning the project. If the wait runs out, the request computes it itself.+ fetcher: async () => {+ const lock = await keyStore+ .acquireLock(+ [KeyStorePrefixes.SecretReadActivityUnusedAccessLock(projectId)],+ UNUSED_ACCESS_COMPUTE_LOCK_TTL_MS,+ {+ retryCount: UNUSED_ACCESS_COMPUTE_WAIT_SECONDS,+ retryDelay: 1000+ }+ )+ .catch(() => undefined);+ try {+ const cached = lock ? await keyStore.getItem(cacheKey) : null;+ if (cached) return JSON.parse(cached) as TUnusedAccessSummary;++ const result = await $computeUnusedAccess(projectId, orgId);+ // Written before the lock is released, so the next request in line finds it. A failed write only costs+ // the next request a recomputation, so it never fails this one.+ await keyStore+ .setItemWithExpiry(cacheKey, UNUSED_ACCESS_CACHE_TTL_SECONDS, JSON.stringify(result))+ .catch((error) => {+ logger.warn(error, `unused-access: failed to cache the project computation [projectId=${projectId}]`);+ });+ return result;+ } finally {+ await lock?.release();+ }+ }});const [remainingTTL, { project, trackingStartedAt }] = await Promise.all([getCacheTtl(keyStore, cacheKey),@@ -721,35 +557,6 @@ export const secretReadActivityServiceFactory = ({};};- // The cache is shared by everyone in the project and can be a day old, so each listed secret is checked against- // the secret as it is now: deleted secrets drop out, keys and paths are current, and a caller only gets the- // secrets they could see anyway.- const $loadVisibleSecrets = async (projectId: string, permission: TSecretPermission, secretIds: string[]) => {- const [tree, currentSecrets] = await Promise.all([- loadProjectFolderTree({ folderDAL, projectEnvDAL }, projectId),- secretIds.length- ? secretV2BridgeDAL.find({ $in: { [`${TableName.SecretV2}.id` as "id"]: [...new Set(secretIds)] } })- : []- ]);- const visible = new Map<string, TUnusedSecret>();- currentSecrets.forEach((secret) => {- const location = tree.locationOf(secret.folderId);- if (!location) return;- const canDescribe = hasSecretReadValueOrDescribePermission(- permission,- ProjectPermissionSecretActions.DescribeSecret,- {- environment: location.environment,- secretPath: location.secretPath,- secretName: secret.key,- secretTags: secret.tags?.map((tag) => tag.slug)- }- );- if (canDescribe) visible.set(secret.id, { secretKey: secret.key, ...location });- });- return visible;- };-const listUnusedAccess = async ({ projectId, offset, limit, search }: TListUnusedAccessDTO,actor: OrgServiceActor@@ -759,7 +566,7 @@ export const secretReadActivityServiceFactory = ({const needle = search?.trim().toLowerCase();const matching = needle ? rows.filter((row) => row.name.toLowerCase().includes(needle)) : rows;return {- principals: matching.slice(offset, offset + limit),+ actors: matching.slice(offset, offset + limit),totalCount: matching.length,projectName: tracking.projectName,trackingStartedAt: tracking.trackingStartedAt,@@ -768,50 +575,48 @@ export const secretReadActivityServiceFactory = ({};};- // The unread secrets of one principal who was not among those cached in full: one scan, kept for the same day.- const $computeUnusedSecretIdsFor = async (- projectId: string,- orgId: string,- principalType: UnusedAccessPrincipalType,- principalId: string- ) => {- const secretIds: string[] = [];- await $walkSecrets(projectId, orgId, (activity, summary) => {- const isUnread =- principalType === UnusedAccessPrincipalType.Group- ? summary.unreadGroups.some((group) => group.id === principalId)- : summary.unread.some(- (reader) => reader.id === principalId && toPrincipalType(reader.actorType) === principalType- );- if (isUnread && secretIds.length < UNUSED_ACCESS_MAX_SECRETS_PER_PRINCIPAL) secretIds.push(activity.secret.id);- });- return secretIds;- };-- const getUnusedAccessPrincipal = async (- { projectId, principalType, principalId }: TGetUnusedAccessPrincipalDTO,+ const getUnusedAccessActor = async (+ { projectId, actorType, actorId }: TGetUnusedAccessActorDTO,actor: OrgServiceActor) => {const { permission } = await checkInsightsReadPermission(permissionService, projectId, actor);const { rows, details, ...tracking } = await $getCachedUnusedAccess(projectId, actor.orgId);- const key = principalKey(principalType, principalId);- const principal = rows.find((row) => principalKey(row.principalType, row.principalId) === key);- if (!principal) {+ const unusedAccessActor = rows.find((row) => row.actorType === actorType && row.actorId === actorId);+ if (!unusedAccessActor) {throw new NotFoundError({- message: `No unused access found for ${principalType} '${principalId}' in this project`+ message: `No unused access found for ${actorType} '${actorId}' in this project`});}+ const key = toActorKey(actorType, actorId);+ // An actor past those cached in full is scanned for on its own, and kept for the same day.const secretIds =details[key] ??(await withCache({keyStore,key: KeyStorePrefixes.InsightsCache(projectId, `unused-access:${key}`),ttlSeconds: UNUSED_ACCESS_CACHE_TTL_SECONDS,- fetcher: () => $computeUnusedSecretIdsFor(projectId, actor.orgId, principalType, principalId)+ fetcher: async () => (await $tallyUnusedAccess(projectId, actor.orgId)).tallies.get(key)?.unreadSecretIds ?? []}));- const visible = await $loadVisibleSecrets(projectId, permission, secretIds);- const secrets = secretIds- .flatMap((secretId) => visible.get(secretId) ?? [])++ // The cache is shared by everyone in the project and can be a day old, so each listed secret is checked against+ // the secret as it is now: deleted secrets drop out, keys and paths are current, and a caller only gets the+ // secrets they could see anyway.+ const [tree, currentSecrets] = await Promise.all([+ loadProjectFolderTree({ folderDAL, projectEnvDAL }, projectId),+ secretIds.length ? secretV2BridgeDAL.find({ $in: { [`${TableName.SecretV2}.id` as "id"]: secretIds } }) : []+ ]);+ const secrets = currentSecrets+ .flatMap((secret): TUnusedSecret[] => {+ const location = tree.locationOf(secret.folderId);+ const canDescribe =+ location &&+ hasSecretReadValueOrDescribePermission(permission, ProjectPermissionSecretActions.DescribeSecret, {+ ...location,+ secretName: secret.key,+ secretTags: secret.tags?.map((tag) => tag.slug)+ });+ return location && canDescribe ? [{ secretKey: secret.key, ...location }] : [];+ }).sort((a, b) =>a.environment.localeCompare(b.environment) ||@@ -819,10 +624,10 @@ export const secretReadActivityServiceFactory = ({a.secretKey.localeCompare(b.secretKey));return {- principal,+ actor: unusedAccessActor,secrets,// Everything not in the list: past the cached cap, hidden from this viewer, or deleted since.- unlistedCount: Math.max(0, principal.unreadCount - secrets.length),+ unlistedCount: Math.max(0, unusedAccessActor.unreadCount - secrets.length),projectName: tracking.projectName,trackingStartedAt: tracking.trackingStartedAt,fullWindowFrom: tracking.fullWindowFrom,@@ -835,6 +640,6 @@ export const secretReadActivityServiceFactory = ({recordBulkRead: secretReadActivityQueue.recordBulkRead,getSecretReadActivity,listUnusedAccess,- getUnusedAccessPrincipal+ getUnusedAccessActor};};