Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.
TypeScript acmeagentsai-agentscertificate-managementdevsecopsenvironment-variableskubernetesopen-sourcepampkipostgresprivate-caprivileged-access-managementsecret-managementsecret-managersecretssecrets-managementsecuritysecurity-toolsvault
revamp db tables for secret scanning
Matheus Nogueirabce5e01fd184
diff --git a/backend/src/ee/routes/v2/secret-scanning-v2-routers/secret-scanning-v2-endpoints.ts b/backend/src/ee/routes/v2/secret-scanning-v2-routers/secret-scanning-v2-endpoints.tsindex 3e2e941513..18a16a2cda 100644--- a/backend/src/ee/routes/v2/secret-scanning-v2-routers/secret-scanning-v2-endpoints.ts+++ b/backend/src/ee/routes/v2/secret-scanning-v2-routers/secret-scanning-v2-endpoints.ts@@ -1,12 +1,15 @@import { z } from "zod";-import { SecretScanningResourcesSchema, SecretScanningScansSchema } from "@app/db/schemas";import { EventType } from "@app/ee/services/audit-log/audit-log-types";import {SecretScanningDataSource,SecretScanningScanStatus} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";import { SECRET_SCANNING_DATA_SOURCE_NAME_MAP } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-maps";+import {+ SecretScanningResourceSchema,+ SecretScanningScanSchema+} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-schemas";import {TSecretScanningDataSource,TSecretScanningDataSourceInput@@ -34,7 +37,7 @@ export const registerSecretScanningEndpoints = <createSchema: z.ZodType<{name: string;projectId: string;- connectionId?: string;+ appConnectionId?: string;config: Partial<I["config"]>;description?: string | null;isAutoScanEnabled?: boolean;@@ -444,7 +447,7 @@ export const registerSecretScanningEndpoints = <dataSourceId: z.string().uuid().describe(SecretScanningDataSources.LIST_RESOURCES(type).dataSourceId)}),response: {- 200: z.object({ resources: SecretScanningResourcesSchema.array() })+ 200: z.object({ resources: SecretScanningResourceSchema.array() })}},onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN, AuthMode.OAUTH]),@@ -489,7 +492,7 @@ export const registerSecretScanningEndpoints = <dataSourceId: z.string().uuid().describe(SecretScanningDataSources.LIST_SCANS(type).dataSourceId)}),response: {- 200: z.object({ scans: SecretScanningScansSchema.array() })+ 200: z.object({ scans: SecretScanningScanSchema.array() })}},onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN, AuthMode.OAUTH]),@@ -533,7 +536,7 @@ export const registerSecretScanningEndpoints = <}),response: {200: z.object({- resources: SecretScanningResourcesSchema.extend({+ resources: SecretScanningResourceSchema.extend({lastScannedAt: z.date().nullish(),lastScanStatus: z.nativeEnum(SecretScanningScanStatus).nullish(),lastScanStatusMessage: z.string().nullish(),@@ -584,7 +587,7 @@ export const registerSecretScanningEndpoints = <}),response: {200: z.object({- scans: SecretScanningScansSchema.extend({+ scans: SecretScanningScanSchema.extend({unresolvedFindings: z.number(),resolvedFindings: z.number(),resourceName: z.string()diff --git a/backend/src/ee/routes/v2/secret-scanning-v2-routers/secret-scanning-v2-router.ts b/backend/src/ee/routes/v2/secret-scanning-v2-routers/secret-scanning-v2-router.tsindex a880131773..6ad3ad6d5e 100644--- a/backend/src/ee/routes/v2/secret-scanning-v2-routers/secret-scanning-v2-router.ts+++ b/backend/src/ee/routes/v2/secret-scanning-v2-routers/secret-scanning-v2-router.ts@@ -164,7 +164,7 @@ export const registerSecretScanningV2Router = async (server: FastifyZodProvider)}),body: z.object({status: z.nativeEnum(SecretScanningFindingStatus).optional().describe(SecretScanningFindings.UPDATE.status),- remarks: z.string().nullish().describe(SecretScanningFindings.UPDATE.remarks)+ triageComment: z.string().trim().max(1024).nullish().describe(SecretScanningFindings.UPDATE.triageComment)}),response: {200: z.object({ finding: SecretScanningFindingSchema })@@ -225,7 +225,7 @@ export const registerSecretScanningV2Router = async (server: FastifyZodProvider).object({findingId: z.string().trim().min(1, "Finding ID required").describe(SecretScanningFindings.UPDATE.findingId),status: z.nativeEnum(SecretScanningFindingStatus).optional().describe(SecretScanningFindings.UPDATE.status),- remarks: z.string().nullish().describe(SecretScanningFindings.UPDATE.remarks)+ triageComment: z.string().trim().max(1024).nullish().describe(SecretScanningFindings.UPDATE.triageComment)}).array().max(500),diff --git a/backend/src/ee/services/secret-scanning-v2/bitbucket/bitbucket-secret-scanning-factory.ts b/backend/src/ee/services/secret-scanning-v2/bitbucket/bitbucket-secret-scanning-factory.tsindex 8804a8de55..65f592caa0 100644--- a/backend/src/ee/services/secret-scanning-v2/bitbucket/bitbucket-secret-scanning-factory.ts+++ b/backend/src/ee/services/secret-scanning-v2/bitbucket/bitbucket-secret-scanning-factory.ts@@ -3,10 +3,7 @@ import RE2 from "re2";import { scanContentAndGetFindings } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-fns";import { SecretMatch } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";-import {- SecretScanningFindingSeverity,- SecretScanningResource-} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";+import { SecretScanningFindingSeverity } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";import {assertProviderRepositorySizeWithinLimit,cloneRepository,@@ -157,8 +154,7 @@ export const BitbucketSecretScanningFactory = () => {return filteredRepos.map(({ full_name, uuid }) => ({name: full_name,- externalId: uuid,- type: SecretScanningResource.Repository+ externalId: uuid}));};@@ -213,8 +209,7 @@ export const BitbucketSecretScanningFactory = () => {> = ({ repository }) => {return {name: repository.full_name,- externalId: repository.uuid,- type: SecretScanningResource.Repository+ externalId: repository.uuid};};@@ -326,7 +321,7 @@ export const BitbucketSecretScanningFactory = () => {details: toFindingDetails(finding),fingerprint: finding.Fingerprint,severity: SecretScanningFindingSeverity.High,- rule: finding.RuleID+ ruleKey: finding.RuleID}));};diff --git a/backend/src/ee/services/secret-scanning-v2/bitbucket/bitbucket-secret-scanning-schemas.ts b/backend/src/ee/services/secret-scanning-v2/bitbucket/bitbucket-secret-scanning-schemas.tsindex 36d036f154..75c1d9d05d 100644--- a/backend/src/ee/services/secret-scanning-v2/bitbucket/bitbucket-secret-scanning-schemas.ts+++ b/backend/src/ee/services/secret-scanning-v2/bitbucket/bitbucket-secret-scanning-schemas.ts@@ -1,9 +1,6 @@import { z } from "zod";-import {- SecretScanningDataSource,- SecretScanningResource-} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";+import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";import {BaseCreateSecretScanningDataSourceSchema,BaseSecretScanningDataSourceSchema,@@ -86,7 +83,6 @@ export const BitbucketDataSourceListItemSchema = z);export const BitbucketFindingSchema = BaseSecretScanningFindingSchema.extend({- resourceType: z.literal(SecretScanningResource.Repository),dataSourceType: z.literal(SecretScanningDataSource.Bitbucket),details: GitRepositoryScanFindingDetailsSchema});diff --git a/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-factory.ts b/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-factory.tsindex 97aae68004..eaf1d48179 100644--- a/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-factory.ts+++ b/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-factory.ts@@ -5,8 +5,7 @@ import { scanContentAndGetFindings } from "@app/ee/services/secret-scanning/secrimport { SecretMatch } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";import {SecretScanningDataSource,- SecretScanningFindingSeverity,- SecretScanningResource+ SecretScanningFindingSeverity} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";import {assertProviderRepositorySizeWithinLimit,@@ -95,8 +94,7 @@ export const GitHubSecretScanningFactory = () => {return filteredRepos.map(({ id, full_name }) => ({name: full_name,- externalId: id.toString(),- type: SecretScanningResource.Repository+ externalId: id.toString()}));};@@ -154,8 +152,7 @@ export const GitHubSecretScanningFactory = () => {> = ({ repository }) => {return {name: repository.full_name,- externalId: repository.id.toString(),- type: SecretScanningResource.Repository+ externalId: repository.id.toString()};};@@ -241,7 +238,7 @@ export const GitHubSecretScanningFactory = () => {details: toFindingDetails(finding),fingerprint: finding.Fingerprint,severity: SecretScanningFindingSeverity.High,- rule: finding.RuleID+ ruleKey: finding.RuleID}));};diff --git a/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-schemas.ts b/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-schemas.tsindex e39830f220..fbe2215dce 100644--- a/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-schemas.ts+++ b/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-schemas.ts@@ -1,9 +1,6 @@import { z } from "zod";-import {- SecretScanningDataSource,- SecretScanningResource-} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";+import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";import {BaseCreateSecretScanningDataSourceSchema,BaseSecretScanningDataSourceSchema,@@ -79,7 +76,6 @@ export const GitHubDataSourceListItemSchema = z);export const GitHubFindingSchema = BaseSecretScanningFindingSchema.extend({- resourceType: z.literal(SecretScanningResource.Repository),dataSourceType: z.literal(SecretScanningDataSource.GitHub),details: GitRepositoryScanFindingDetailsSchema});diff --git a/backend/src/ee/services/secret-scanning-v2/gitlab/gitlab-secret-scanning-factory.ts b/backend/src/ee/services/secret-scanning-v2/gitlab/gitlab-secret-scanning-factory.tsindex 443e4a1879..b0257a8935 100644--- a/backend/src/ee/services/secret-scanning-v2/gitlab/gitlab-secret-scanning-factory.ts+++ b/backend/src/ee/services/secret-scanning-v2/gitlab/gitlab-secret-scanning-factory.ts@@ -3,10 +3,7 @@ import { join } from "path";import { scanContentAndGetFindings } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-fns";import { SecretMatch } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";-import {- SecretScanningFindingSeverity,- SecretScanningResource-} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";+import { SecretScanningFindingSeverity } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";import {assertProviderRepositorySizeWithinLimit,cloneRepository,@@ -214,8 +211,7 @@ export const GitLabSecretScanningFactory = ({ appConnectionDAL, kmsService }: TSreturn [{name: project.pathWithNamespace,- externalId: project.id.toString(),- type: SecretScanningResource.Project+ externalId: project.id.toString()}];}@@ -237,8 +233,7 @@ export const GitLabSecretScanningFactory = ({ appConnectionDAL, kmsService }: TSreturn filteredProjects.map(({ id, pathWithNamespace }) => ({name: pathWithNamespace,- externalId: id.toString(),- type: SecretScanningResource.Project+ externalId: id.toString()}));};@@ -308,8 +303,7 @@ export const GitLabSecretScanningFactory = ({ appConnectionDAL, kmsService }: TS> = ({ project }) => {return {name: project.path_with_namespace,- externalId: project.id.toString(),- type: SecretScanningResource.Project+ externalId: project.id.toString()};};@@ -378,7 +372,7 @@ export const GitLabSecretScanningFactory = ({ appConnectionDAL, kmsService }: TSdetails: toFindingDetails(finding),fingerprint: finding.Fingerprint,severity: SecretScanningFindingSeverity.High,- rule: finding.RuleID+ ruleKey: finding.RuleID}));};diff --git a/backend/src/ee/services/secret-scanning-v2/gitlab/gitlab-secret-scanning-schemas.ts b/backend/src/ee/services/secret-scanning-v2/gitlab/gitlab-secret-scanning-schemas.tsindex e70e63c11c..1c91037ab2 100644--- a/backend/src/ee/services/secret-scanning-v2/gitlab/gitlab-secret-scanning-schemas.ts+++ b/backend/src/ee/services/secret-scanning-v2/gitlab/gitlab-secret-scanning-schemas.ts@@ -1,10 +1,7 @@import { z } from "zod";import { GitLabDataSourceScope } from "@app/ee/services/secret-scanning-v2/gitlab/gitlab-secret-scanning-enums";-import {- SecretScanningDataSource,- SecretScanningResource-} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";+import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";import {BaseCreateSecretScanningDataSourceSchema,BaseSecretScanningDataSourceSchema,@@ -90,7 +87,6 @@ export const GitLabDataSourceListItemSchema = z);export const GitLabFindingSchema = BaseSecretScanningFindingSchema.extend({- resourceType: z.literal(SecretScanningResource.Project),dataSourceType: z.literal(SecretScanningDataSource.GitLab),details: GitRepositoryScanFindingDetailsSchema});diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-dal.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-dal.tsindex aadc6dc83e..34de7627c7 100644--- a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-dal.ts+++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-dal.ts@@ -6,7 +6,9 @@ import {SecretScanningScansSchema,TableName,TSecretScanningDataSources,- TSecretScanningFindings+ TSecretScanningFindings,+ TSecretScanningResources,+ TSecretScanningResourcesInsert} from "@app/db/schemas";import {SecretScanningFindingStatus,@@ -41,9 +43,10 @@ const baseSecretScanningDataSourceQuery = ({const query = (tx || db.replicaNode())(TableName.SecretScanningDataSource).join(TableName.AppConnection,- `${TableName.SecretScanningDataSource}.connectionId`,+ `${TableName.SecretScanningDataSource}.appConnectionId`,`${TableName.AppConnection}.id`)+ .whereNull(`${TableName.SecretScanningDataSource}.deletedAt`).select(selectAllTableCols(TableName.SecretScanningDataSource)).select(// entire connection@@ -82,7 +85,7 @@ const expandSecretScanningDataSource = <const {connectionApp,connectionName,- connectionId,+ appConnectionId,connectionOrgId,connectionIsAutoRotationEnabled,connectionEncryptedCredentials,@@ -100,11 +103,11 @@ const expandSecretScanningDataSource = <return {...el,- connectionId,- connection: connectionId+ appConnectionId,+ connection: appConnectionId? {app: connectionApp,- id: connectionId,+ id: appConnectionId,name: connectionName,orgId: connectionOrgId,encryptedCredentials: connectionEncryptedCredentials,@@ -200,6 +203,8 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {return expandSecretScanningDataSource(dataSource);};+ // Soft delete: the source's findings stay in the database for audit, but every read path filters+ // deleted sources out, and the partial unique index frees the name for reuse.const deleteDataSourceById = async (dataSourceId: string, tx?: Knex) => {const dataSource = (await baseSecretScanningDataSourceQuery({filter: { id: dataSourceId },@@ -207,7 +212,7 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {tx}).first())!;- await dataSourceOrm.deleteById(dataSourceId, tx);+ await dataSourceOrm.updateById(dataSourceId, { deletedAt: new Date() }, tx);return expandSecretScanningDataSource(dataSource);};@@ -229,11 +234,11 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {// TODO (scott): this query will probably need to be optimizedconst dataSources = await baseSecretScanningDataSourceQuery({ filter, db, tx })- .leftJoin(- TableName.SecretScanningResource,- `${TableName.SecretScanningResource}.dataSourceId`,- `${TableName.SecretScanningDataSource}.id`- )+ .leftJoin(TableName.SecretScanningResource, (qb) => {+ void qb+ .on(`${TableName.SecretScanningResource}.sourceId`, `${TableName.SecretScanningDataSource}.id`)+ .andOnNull(`${TableName.SecretScanningResource}.deletedAt`);+ }).leftJoin(TableName.SecretScanningScan,`${TableName.SecretScanningScan}.resourceId`,@@ -315,6 +320,7 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {if (filter)void qb.where(buildFindFilter(prependTableNameToFindFilter(TableName.SecretScanningResource, filter)));})+ .whereNull(`${TableName.SecretScanningResource}.deletedAt`).leftJoin(TableName.SecretScanningScan,`${TableName.SecretScanningScan}.resourceId`,@@ -400,7 +406,8 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {`${TableName.SecretScanningResource}.id`,`${TableName.SecretScanningScan}.resourceId`)- .where(`${TableName.SecretScanningResource}.dataSourceId`, dataSourceId)+ .where(`${TableName.SecretScanningResource}.sourceId`, dataSourceId)+ .whereNull(`${TableName.SecretScanningResource}.deletedAt`).leftJoin(TableName.SecretScanningFinding,`${TableName.SecretScanningFinding}.scanId`,@@ -466,7 +473,7 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {// time so they're reaped too..whereRaw(`COALESCE(??, ??, ??) < ?`, [`${TableName.SecretScanningScan}.progressUpdatedAt`,- `${TableName.SecretScanningScan}.scanningStartedAt`,+ `${TableName.SecretScanningScan}.startedAt`,`${TableName.SecretScanningScan}.createdAt`,startedBefore])@@ -478,8 +485,7 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {.select(selectAllTableCols(TableName.SecretScanningScan)).select(db.ref("name").withSchema(TableName.SecretScanningResource).as("resourceName"),- db.ref("type").withSchema(TableName.SecretScanningResource).as("resourceType"),- db.ref("dataSourceId").withSchema(TableName.SecretScanningResource)+ db.ref("sourceId").withSchema(TableName.SecretScanningResource).as("dataSourceId")).orderBy(`${TableName.SecretScanningScan}.createdAt`, "asc").limit(limit);@@ -498,8 +504,8 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {`${TableName.SecretScanningResource}.id`,`${TableName.SecretScanningScan}.resourceId`)- .where(`${TableName.SecretScanningResource}.dataSourceId`, dataSourceId)-+ .where(`${TableName.SecretScanningResource}.sourceId`, dataSourceId)+ .whereNull(`${TableName.SecretScanningResource}.deletedAt`).select(selectAllTableCols(TableName.SecretScanningScan));return scans;@@ -508,6 +514,103 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {}};+ // The unique index on (sourceId, externalId) is partial (active rows only), and an ON CONFLICT+ // target has to repeat that predicate to match it, which ormify's upsert cannot express.+ const upsertResources = async (data: TSecretScanningResourcesInsert[], tx?: Knex) => {+ if (!data.length) return [];++ try {+ const { sql, bindings } = (tx || db)(TableName.SecretScanningResource).insert(data).toSQL();++ const result = (await (tx || db).raw(+ `${sql} ON CONFLICT ("sourceId", "externalId") WHERE "deletedAt" IS NULL DO UPDATE SET "name" = EXCLUDED."name" RETURNING *`,+ bindings+ )) as { rows: TSecretScanningResources[] };++ return result.rows;+ } catch (error) {+ throw new DatabaseError({ error, name: "Upsert - Secret Scanning Resource" });+ }+ };++ const baseFindingQuery = (tx?: Knex) =>+ (tx || db.replicaNode())(TableName.SecretScanningFinding)+ .join(+ TableName.SecretScanningResource,+ `${TableName.SecretScanningResource}.id`,+ `${TableName.SecretScanningFinding}.resourceId`+ )+ .join(+ TableName.SecretScanningDataSource,+ `${TableName.SecretScanningDataSource}.id`,+ `${TableName.SecretScanningResource}.sourceId`+ )+ .whereNull(`${TableName.SecretScanningResource}.deletedAt`)+ .whereNull(`${TableName.SecretScanningDataSource}.deletedAt`);++ const selectFindingWithDetails = <T extends Knex.QueryBuilder>(query: T) =>+ query+ .select(selectAllTableCols(TableName.SecretScanningFinding))+ .select(+ db.ref("projectId").withSchema(TableName.SecretScanningDataSource),+ db.ref("id").withSchema(TableName.SecretScanningDataSource).as("dataSourceId"),+ db.ref("name").withSchema(TableName.SecretScanningDataSource).as("dataSourceName"),+ db.ref("type").withSchema(TableName.SecretScanningDataSource).as("dataSourceType"),+ db.ref("name").withSchema(TableName.SecretScanningResource).as("resourceName")+ );++ type TSecretScanningFindingWithDetails = TSecretScanningFindings & {+ projectId: string;+ dataSourceId: string;+ dataSourceName: string;+ dataSourceType: string;+ resourceName: string;+ };++ const findFindingsByProjectId = async (projectId: string, tx?: Knex) => {+ try {+ const findings = (await selectFindingWithDetails(+ baseFindingQuery(tx).where(`${TableName.SecretScanningDataSource}.projectId`, projectId)+ )) as TSecretScanningFindingWithDetails[];++ return findings;+ } catch (error) {+ throw new DatabaseError({ error, name: "Find By Project ID - Secret Scanning Finding" });+ }+ };++ const findFindingByIdWithDetails = async (findingId: string, tx?: Knex) => {+ try {+ const finding = (await selectFindingWithDetails(+ baseFindingQuery(tx).where(`${TableName.SecretScanningFinding}.id`, findingId)+ ).first()) as TSecretScanningFindingWithDetails | undefined;++ return finding;+ } catch (error) {+ throw new DatabaseError({ error, name: "Find By ID - Secret Scanning Finding" });+ }+ };++ const countFindingsByProjectId = async (+ projectId: string,+ filter: { status?: SecretScanningFindingStatus } = {},+ tx?: Knex+ ) => {+ try {+ const query = baseFindingQuery(tx).where(`${TableName.SecretScanningDataSource}.projectId`, projectId);++ if (filter.status) void query.where(`${TableName.SecretScanningFinding}.status`, filter.status);++ const result = (await query.count(`${TableName.SecretScanningFinding}.id as count`).first()) as+ | { count: string }+ | undefined;++ return Number(result?.count ?? 0);+ } catch (error) {+ throw new DatabaseError({ error, name: "Count By Project ID - Secret Scanning Finding" });+ }+ };+/*** Counts the findings this scan was the first to discover, which is the number the scan's own* detail view shows. It is read back from the database rather than accumulated in the handler@@ -541,6 +644,7 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {},resources: {...resourceOrm,+ upsert: upsertResources,findWithDetails: findResourcesWithDetails},scans: {@@ -552,7 +656,10 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {findings: {...findingOrm,upsert: upsertFindings,- countByScanId: countFindingsByScanId+ countByScanId: countFindingsByScanId,+ findByProjectId: findFindingsByProjectId,+ findByIdWithDetails: findFindingByIdWithDetails,+ countByProjectId: countFindingsByProjectId},configs: configOrm};diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-enums.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-enums.tsindex a4fd7bd34a..b2ce9f2524 100644--- a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-enums.ts+++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-enums.ts@@ -12,8 +12,13 @@ export enum SecretScanningScanStatus {}export enum SecretScanningScanType {- FullScan = "full-scan",- DiffScan = "diff-scan"+ Historical = "historical",+ Realtime = "realtime"+}++export enum SecretScanningScanTrigger {+ Manual = "manual",+ Push = "push"}export enum SecretScanningFindingStatus {@@ -33,3 +38,9 @@ export enum SecretScanningFindingSeverity {Medium = "medium",Low = "low"}++export enum SecretScanningFindingConfidence {+ High = "high",+ Medium = "medium",+ Low = "low"+}diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.tsindex 966117c0da..0d3cd0184b 100644--- a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.ts+++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.ts@@ -360,7 +360,7 @@ export const scanGitRepositoryAndGetFindings = async (details: toFindingDetails(finding),fingerprint: `${finding.Fingerprint}:${finding.StartColumn}`,severity: SecretScanningFindingSeverity.High,- rule: finding.RuleID+ ruleKey: finding.RuleID}));};diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-maps.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-maps.tsindex fd29e0b74f..c514946dea 100644--- a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-maps.ts+++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-maps.ts@@ -1,4 +1,7 @@-import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";+import {+ SecretScanningDataSource,+ SecretScanningResource+} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";import { AppConnection } from "@app/services/app-connection/app-connection-enums";export const SECRET_SCANNING_DATA_SOURCE_NAME_MAP: Record<SecretScanningDataSource, string> = {@@ -18,3 +21,9 @@ export const AUTO_SYNC_DESCRIPTION_HELPER: Record<SecretScanningDataSource, { ve[SecretScanningDataSource.Bitbucket]: { verb: "push", noun: "repositories" },[SecretScanningDataSource.GitLab]: { verb: "push", noun: "projects" }};++export const SECRET_SCANNING_DATA_SOURCE_RESOURCE_TYPE_MAP: Record<SecretScanningDataSource, SecretScanningResource> = {+ [SecretScanningDataSource.GitHub]: SecretScanningResource.Repository,+ [SecretScanningDataSource.Bitbucket]: SecretScanningResource.Repository,+ [SecretScanningDataSource.GitLab]: SecretScanningResource.Project+};diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.tsindex e1f6e1f450..73cc3e528b 100644--- a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts+++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts@@ -36,9 +36,11 @@ import {SecretScanningDataSource,SecretScanningResource,SecretScanningScanStatus,+ SecretScanningScanTrigger,SecretScanningScanType} from "./secret-scanning-v2-enums";import { SECRET_SCANNING_FACTORY_MAP } from "./secret-scanning-v2-factory";+import { SECRET_SCANNING_DATA_SOURCE_RESOURCE_TYPE_MAP } from "./secret-scanning-v2-maps";import {TFindingsPayload,TQueueSecretScanningDataSourceFullScan,@@ -139,7 +141,7 @@ export const secretScanningV2QueueServiceFactory = ({},{status: SecretScanningScanStatus.Scanning,- scanningStartedAt: new Date(),+ startedAt: new Date(),...(trackProgress ? { progressUpdatedAt: new Date() } : {})});@@ -160,7 +162,8 @@ export const secretScanningV2QueueServiceFactory = ({},{status: SecretScanningScanStatus.Failed,- statusMessage+ statusMessage,+ completedAt: new Date()});@@ -169,7 +172,7 @@ export const secretScanningV2QueueServiceFactory = ({const queueDataSourceFullScan = async (dataSource: TSecretScanningDataSourceWithConnection,- resourceExternalId?: string+ { resourceExternalId, triggeredByUserId }: { resourceExternalId?: string; triggeredByUserId?: string } = {}) => {try {const { type } = dataSource;@@ -198,15 +201,14 @@ export const secretScanningV2QueueServiceFactory = ({const resources = await secretScanningV2DAL.resources.upsert(filteredRawResources.map((rawResource) => ({...rawResource,- dataSourceId: dataSource.id+ sourceId: dataSource.id})),- ["externalId", "dataSourceId"],tx);const inFlightScans = await secretScanningV2DAL.scans.find({- type: SecretScanningScanType.FullScan,+ type: SecretScanningScanType.Historical,$in: {resourceId: resources.map((resource) => resource.id),status: [SecretScanningScanStatus.Queued, SecretScanningScanStatus.Scanning]@@ -223,7 +225,9 @@ export const secretScanningV2QueueServiceFactory = ({const scans = await secretScanningV2DAL.scans.insertMany(resourcesToScan.map((resource) => ({resourceId: resource.id,- type: SecretScanningScanType.FullScan+ type: SecretScanningScanType.Historical,+ trigger: SecretScanningScanTrigger.Manual,+ triggeredByUserId: triggeredByUserId ?? null})),tx);@@ -273,7 +277,7 @@ export const secretScanningV2QueueServiceFactory = ({// Logged before any work so a `ps` on a saturated worker can be tied back to a scan ID.logger.info(- `secretScanningV2Queue: Full Scan Started ${logDetails} [scanType=${SecretScanningScanType.FullScan}] [tempFolder=${tempFolder}]`+ `secretScanningV2Queue: Full Scan Started ${logDetails} [scanType=${SecretScanningScanType.Historical}] [tempFolder=${tempFolder}]`);const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);@@ -288,6 +292,8 @@ export const secretScanningV2QueueServiceFactory = ({if (!scan) throw new Error(`Scan with ID "${scanId}" not found`);+ const resourceType = SECRET_SCANNING_DATA_SOURCE_RESOURCE_TYPE_MAP[dataSource.type as SecretScanningDataSource];+let holdsLease = false;try {@@ -305,7 +311,7 @@ export const secretScanningV2QueueServiceFactory = ({if (!started) {logger.warn(- `secretScanningV2Queue: Full Scan skipped, scan was already closed out ${logDetails} [scanType=${SecretScanningScanType.FullScan}]`+ `secretScanningV2Queue: Full Scan skipped, scan was already closed out ${logDetails} [scanType=${SecretScanningScanType.Historical}]`);return;}@@ -360,16 +366,13 @@ export const secretScanningV2QueueServiceFactory = ({await secretScanningV2DAL.findings.upsert(batchFindings.map((finding) => ({...finding,- projectId: dataSource.projectId,- dataSourceName: dataSource.name,- dataSourceType: dataSource.type,- resourceName: resource.name,- resourceType: resource.type,+ resourceId,scanId})),- ["projectId", "fingerprint"],+ ["resourceId", "fingerprint"],tx,- ["resourceName", "dataSourceName"]+ // a no-op merge, so rows already found by an earlier scan are returned with their original scanId+ ["ruleKey"]);}@@ -397,7 +400,7 @@ export const secretScanningV2QueueServiceFactory = ({let stillOwned = true;- switch (resource.type) {+ switch (resourceType) {case SecretScanningResource.Repository:case SecretScanningResource.Project: {const repoSizeMb = await assertClonedRepositoryWithinSizeLimit(resource.name, scanPath);@@ -462,7 +465,8 @@ export const secretScanningV2QueueServiceFactory = ({{ id: scanId, status: SecretScanningScanStatus.Scanning },{status: SecretScanningScanStatus.Completed,- statusMessage: null+ statusMessage: null,+ completedAt: new Date()}): [];@@ -506,10 +510,10 @@ export const secretScanningV2QueueServiceFactory = ({dataSourceId: dataSource.id,dataSourceType: dataSource.type,resourceId: resource.id,- resourceType: resource.type,+ resourceType,scanId,scanStatus: SecretScanningScanStatus.Completed,- scanType: SecretScanningScanType.FullScan,+ scanType: SecretScanningScanType.Historical,numberOfSecretsDetected: findingsCount}}@@ -549,10 +553,10 @@ export const secretScanningV2QueueServiceFactory = ({dataSourceId: dataSource.id,dataSourceType: dataSource.type,resourceId: resource.id,- resourceType: resource.type,+ resourceType,scanId,scanStatus: SecretScanningScanStatus.Failed,- scanType: SecretScanningScanType.FullScan+ scanType: SecretScanningScanType.Historical}}});@@ -588,17 +592,17 @@ export const secretScanningV2QueueServiceFactory = ({[{...resourcePayload,- dataSourceId+ sourceId: dataSourceId}],- ["externalId", "dataSourceId"],tx);const scan = await secretScanningV2DAL.scans.create({resourceId: resource.id,- type: SecretScanningScanType.DiffScan+ type: SecretScanningScanType.Realtime,+ trigger: SecretScanningScanTrigger.Push},tx);@@ -658,10 +662,12 @@ export const secretScanningV2QueueServiceFactory = ({appConnectionDAL});+ const resourceType = SECRET_SCANNING_DATA_SOURCE_RESOURCE_TYPE_MAP[dataSource.type as SecretScanningDataSource];+const tempFolder = await createTempFolder();logger.info(- `secretScanningV2Queue: Diff Scan Started ${logDetails} [scanType=${SecretScanningScanType.DiffScan}] [tempFolder=${tempFolder}]`+ `secretScanningV2Queue: Diff Scan Started ${logDetails} [scanType=${SecretScanningScanType.Realtime}] [tempFolder=${tempFolder}]`);try {@@ -669,7 +675,7 @@ export const secretScanningV2QueueServiceFactory = ({if (!started) {logger.warn(- `secretScanningV2Queue: Diff Scan skipped, scan was already closed out ${logDetails} [scanType=${SecretScanningScanType.DiffScan}]`+ `secretScanningV2Queue: Diff Scan skipped, scan was already closed out ${logDetails} [scanType=${SecretScanningScanType.Realtime}]`);return;}@@ -705,16 +711,12 @@ export const secretScanningV2QueueServiceFactory = ({findings = await secretScanningV2DAL.findings.upsert(findingsPayload.map((finding) => ({...finding,- projectId: dataSource.projectId,- dataSourceName: dataSource.name,- dataSourceType: dataSource.type,- resourceName: resource.name,- resourceType: resource.type,+ resourceId,scanId})),- ["projectId", "fingerprint"],+ ["resourceId", "fingerprint"],tx,- ["resourceName", "dataSourceName"]+ ["ruleKey"]);}@@ -722,7 +724,8 @@ export const secretScanningV2QueueServiceFactory = ({const completedScans = await secretScanningV2DAL.scans.update({ id: scanId, status: SecretScanningScanStatus.Scanning },{- status: SecretScanningScanStatus.Completed+ status: SecretScanningScanStatus.Completed,+ completedAt: new Date()},tx);@@ -770,10 +773,10 @@ export const secretScanningV2QueueServiceFactory = ({dataSourceId: dataSource.id,dataSourceType: dataSource.type,resourceId,- resourceType: resource.type,+ resourceType,scanId,scanStatus: SecretScanningScanStatus.Completed,- scanType: SecretScanningScanType.DiffScan,+ scanType: SecretScanningScanType.Realtime,numberOfSecretsDetected: findingsPayload.length}}@@ -813,10 +816,10 @@ export const secretScanningV2QueueServiceFactory = ({dataSourceId: dataSource.id,dataSourceType: dataSource.type,resourceId: resource.id,- resourceType: resource.type,+ resourceType,scanId,scanStatus: SecretScanningScanStatus.Failed,- scanType: SecretScanningScanType.DiffScan+ scanType: SecretScanningScanType.Realtime}}});@@ -929,7 +932,7 @@ export const secretScanningV2QueueServiceFactory = ({// so the row it set to `scanning` stays that way forever and the customer sees a scan permanently// in progress.const failStuckScan = async (scan: Awaited<ReturnType<typeof secretScanningV2DAL.scans.findStuck>>[number]) => {- const logDetails = `[scanId=${scan.id}] [resourceId=${scan.resourceId}] [dataSourceId=${scan.dataSourceId}] [scanningStartedAt=${scan.scanningStartedAt?.toISOString()}]`;+ const logDetails = `[scanId=${scan.id}] [resourceId=${scan.resourceId}] [dataSourceId=${scan.dataSourceId}] [startedAt=${scan.startedAt?.toISOString()}]`;try {// Guarded on the status we read: if the worker did finish between the read and this write, the@@ -938,7 +941,8 @@ export const secretScanningV2QueueServiceFactory = ({{ id: scan.id, status: SecretScanningScanStatus.Scanning },{status: SecretScanningScanStatus.Failed,- statusMessage: STUCK_SCAN_STATUS_MESSAGE+ statusMessage: STUCK_SCAN_STATUS_MESSAGE,+ completedAt: new Date()});@@ -974,7 +978,7 @@ export const secretScanningV2QueueServiceFactory = ({dataSourceId: dataSource.id,dataSourceType: dataSource.type,resourceId: scan.resourceId,- resourceType: scan.resourceType,+ resourceType: SECRET_SCANNING_DATA_SOURCE_RESOURCE_TYPE_MAP[dataSource.type],scanId: scan.id,scanStatus: SecretScanningScanStatus.Failed,scanType: scan.type as SecretScanningScanTypediff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-schemas.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-schemas.tsindex 730427a18f..09636ac5a7 100644--- a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-schemas.ts+++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-schemas.ts@@ -1,6 +1,11 @@import { z } from "zod";-import { SecretScanningDataSourcesSchema, SecretScanningFindingsSchema } from "@app/db/schemas";+import {+ SecretScanningDataSourcesSchema,+ SecretScanningFindingsSchema,+ SecretScanningResourcesSchema,+ SecretScanningScansSchema+} from "@app/db/schemas";import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";import { SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-maps";import { SecretScanningDataSources } from "@app/lib/api-docs";@@ -18,11 +23,14 @@ export const BaseSecretScanningDataSourceSchema = ({SecretScanningDataSourcesSchema.omit({// unique to providertype: true,- connectionId: true,- config: true+ appConnectionId: true,+ config: true,+ // internal+ encryptedCredentials: true,+ deletedAt: true}).extend({type: z.literal(type),- connectionId: isConnectionRequired ? z.string().uuid() : z.null(),+ appConnectionId: isConnectionRequired ? z.string().uuid() : z.null(),connection: isConnectionRequired? z.object({app: z.literal(SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP[type]),@@ -49,8 +57,8 @@ export const BaseCreateSecretScanningDataSourceSchema = ({.max(256, "Description cannot exceed 256 characters").nullish().describe(SecretScanningDataSources.CREATE(type).description),- connectionId: isConnectionRequired- ? z.string().uuid().describe(SecretScanningDataSources.CREATE(type).connectionId)+ appConnectionId: isConnectionRequired+ ? z.string().uuid().describe(SecretScanningDataSources.CREATE(type).appConnectionId): z.undefined(),isAutoScanEnabled: z.boolean()@@ -92,7 +100,21 @@ export const GitRepositoryScanFindingDetailsSchema = z.object({});export const BaseSecretScanningFindingSchema = SecretScanningFindingsSchema.omit({- dataSourceType: true,- resourceType: true,- details: true+ details: true,+ encryptedFinding: true+}).extend({+ projectId: z.string(),+ dataSourceId: z.string().uuid(),+ dataSourceName: z.string(),+ resourceName: z.string()+});++export const SecretScanningResourceSchema = SecretScanningResourcesSchema.omit({+ deletedAt: true+});++export const SecretScanningScanSchema = SecretScanningScansSchema.omit({+ lastScannedCommit: true,+ lastScannedCommitDigest: true,+ progressUpdatedAt: true});diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-service.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-service.tsindex 36a425d7b2..9fadc17213 100644--- a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-service.ts+++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-service.ts@@ -47,6 +47,7 @@ import { DatabaseErrorCode } from "@app/lib/error-codes";import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors";import { logger } from "@app/lib/logger";import { OrgServiceActor } from "@app/lib/types";+import { ActorType } from "@app/services/auth/auth-type";import { TAppConnectionDALFactory } from "@app/services/app-connection/app-connection-dal";import { decryptAppConnection } from "@app/services/app-connection/app-connection-fns";import { TAppConnectionServiceFactory } from "@app/services/app-connection/app-connection-service";@@ -251,11 +252,11 @@ export const secretScanningV2ServiceFactory = ({);let connection: TAppConnection | null = null;- if (payload.connectionId) {+ if (payload.appConnectionId) {// validates permission to connect and app is valid for data sourceconnection = await appConnectionService.validateAppConnectionUsageById(SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP[payload.type],- { connectionId: payload.connectionId, projectId: payload.projectId },+ { connectionId: payload.appConnectionId, projectId: payload.projectId },actor);}@@ -312,10 +313,13 @@ export const secretScanningV2ServiceFactory = ({if (payload.isAutoScanEnabled) {try {- await secretScanningV2Queue.queueDataSourceFullScan({- ...createdDataSource,- connection- } as TSecretScanningDataSourceWithConnection);+ await secretScanningV2Queue.queueDataSourceFullScan(+ {+ ...createdDataSource,+ connection+ } as TSecretScanningDataSourceWithConnection,+ { triggeredByUserId: actor.type === ActorType.USER ? actor.id : undefined }+ );} catch (error) {// Deliberately non-fatal — a scan that can't be queued shouldn't block creating the data source.logger.error(@@ -376,11 +380,11 @@ export const secretScanningV2ServiceFactory = ({});let connection: TAppConnection | null = null;- if (dataSource.connectionId) {+ if (dataSource.appConnectionId) {// validates permission to connect and app is valid for data sourceconnection = await appConnectionService.validateAppConnectionUsageById(SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP[dataSource.type],- { connectionId: dataSource.connectionId, projectId: dataSource.projectId },+ { connectionId: dataSource.appConnectionId, projectId: dataSource.projectId },actor);}@@ -536,7 +540,11 @@ export const secretScanningV2ServiceFactory = ({let resourceExternalId: string | undefined;if (resourceId) {- const resource = await secretScanningV2DAL.resources.findOne({ id: resourceId, dataSourceId });+ const resource = await secretScanningV2DAL.resources.findOne({+ id: resourceId,+ sourceId: dataSourceId,+ deletedAt: null+ });if (!resource) {throw new NotFoundError({message: `Could not find Secret Scanning Resource with ID "${resourceId}" for Data Source with ID "${dataSourceId}"`@@ -550,7 +558,7 @@ export const secretScanningV2ServiceFactory = ({...dataSource,connection} as TSecretScanningDataSourceWithConnection,- resourceExternalId+ { resourceExternalId, triggeredByUserId: actor.type === ActorType.USER ? actor.id : undefined });return dataSource as TSecretScanningDataSource;@@ -595,7 +603,8 @@ export const secretScanningV2ServiceFactory = ({});const resources = await secretScanningV2DAL.resources.find({- dataSourceId+ sourceId: dataSourceId,+ deletedAt: null});return { resources, projectId: dataSource.projectId };@@ -682,7 +691,7 @@ export const secretScanningV2ServiceFactory = ({message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}`});- const resources = await secretScanningV2DAL.resources.findWithDetails({ dataSourceId });+ const resources = await secretScanningV2DAL.resources.findWithDetails({ sourceId: dataSourceId });return { resources: resources as TSecretScanningResourceWithDetails[], projectId: dataSource.projectId };};@@ -753,15 +762,9 @@ export const secretScanningV2ServiceFactory = ({ProjectPermissionSub.SecretScanningFindings);- const [finding] = await secretScanningV2DAL.findings.find(- {- projectId,- status: SecretScanningFindingStatus.Unresolved- },- { count: true }- );-- return Number(finding?.count ?? 0);+ return secretScanningV2DAL.findings.countByProjectId(projectId, {+ status: SecretScanningFindingStatus.Unresolved+ });};const listSecretScanningFindingsByProjectId = async (projectId: string, actor: OrgServiceActor) => {@@ -787,15 +790,13 @@ export const secretScanningV2ServiceFactory = ({ProjectPermissionSub.SecretScanningFindings);- const findings = await secretScanningV2DAL.findings.find({- projectId- });+ const findings = await secretScanningV2DAL.findings.findByProjectId(projectId);return findings as TSecretScanningFinding[];};const updateSecretScanningFindingById = async (- { findingId, remarks, status }: TUpdateSecretScanningFindingDTO,+ { findingId, triageComment, status }: TUpdateSecretScanningFindingDTO,actor: OrgServiceActor) => {const plan = await licenseService.getPlan(actor.orgId);@@ -806,7 +807,7 @@ export const secretScanningV2ServiceFactory = ({"Failed to access Secret Scanning Findings due to plan restriction. Upgrade plan to enable Secret Scanning."});- const finding = await secretScanningV2DAL.findings.findById(findingId);+ const finding = await secretScanningV2DAL.findings.findByIdWithDetails(findingId);if (!finding)throw new NotFoundError({@@ -827,9 +828,25 @@ export const secretScanningV2ServiceFactory = ({ProjectPermissionSub.SecretScanningFindings);- const updatedFinding = await secretScanningV2DAL.findings.updateById(findingId, {- remarks,- status+ const isStatusChange = status !== undefined && status !== finding.status;+ const now = new Date();++ const updatedFinding = await secretScanningV2DAL.findings.transaction(async (tx) => {+ await secretScanningV2DAL.findings.updateById(+ findingId,+ {+ triageComment,+ status,+ ...(isStatusChange || triageComment !== undefined+ ? { triagedAt: now, triagedByUserId: actor.type === ActorType.USER ? actor.id : null }+ : {}),+ ...(isStatusChange ? { resolvedAt: status === SecretScanningFindingStatus.Resolved ? now : null } : {})+ },+ tx+ );++ // read back in the same transaction: a replica may not have the write yet+ return secretScanningV2DAL.findings.findByIdWithDetails(findingId, tx);});return { finding: updatedFinding as TSecretScanningFinding, projectId: finding.projectId };diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-types.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-types.tsindex d885b05f5f..33818d52cf 100644--- a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-types.ts+++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-types.ts@@ -103,14 +103,14 @@ export type TCreateSecretScanningDataSourceDTO = Pick<TSecretScanningDataSource,"description" | "name" | "projectId"> & {- connectionId?: string;+ appConnectionId?: string;type: SecretScanningDataSource;isAutoScanEnabled?: boolean;config: Partial<TSecretScanningDataSourceInput["config"]>;};export type TUpdateSecretScanningDataSourceDTO = Partial<- Omit<TCreateSecretScanningDataSourceDTO, "projectId" | "connectionId">+ Omit<TCreateSecretScanningDataSourceDTO, "projectId" | "appConnectionId">> & {dataSourceId: string;type: SecretScanningDataSource;@@ -168,11 +168,11 @@ export type TCloneRepository = {export type TSecretScanningFactoryListRawResources<T extends TSecretScanningDataSourceWithConnection> = (dataSource: T-) => Promise<Pick<TSecretScanningResources, "externalId" | "name" | "type">[]>;+) => Promise<Pick<TSecretScanningResources, "externalId" | "name">[]>;export type TSecretScanningFactoryGetDiffScanResourcePayload<P extends TQueueSecretScanningResourceDiffScan["payload"]-> = (payload: P) => Pick<TSecretScanningResources, "externalId" | "name" | "type">;+> = (payload: P) => Pick<TSecretScanningResources, "externalId" | "name">;export type TSecretScanningFactoryGetFullScanPath<T extends TSecretScanningDataSourceWithConnection> = (parameters: {dataSource: T;@@ -239,12 +239,15 @@ export type TSecretScanningFactory<getDiffScanFindingsPayload: TSecretScanningFactoryGetDiffScanFindingsPayload<T, P>;};-export type TFindingsPayload = Pick<TSecretScanningFindingsInsert, "details" | "fingerprint" | "severity" | "rule">[];+export type TFindingsPayload = Pick<+ TSecretScanningFindingsInsert,+ "details" | "fingerprint" | "severity" | "ruleKey"+>[];export type TGetFindingsPayload = Promise<TFindingsPayload>;export type TUpdateSecretScanningFindingDTO = {status?: SecretScanningFindingStatus;- remarks?: string | null;+ triageComment?: string | null;findingId: string;};diff --git a/backend/src/lib/api-docs/constants.ts b/backend/src/lib/api-docs/constants.tsindex a698fe1802..da3b614ab3 100644--- a/backend/src/lib/api-docs/constants.ts+++ b/backend/src/lib/api-docs/constants.ts@@ -3792,7 +3792,7 @@ export const SecretScanningDataSources = {name: `The name of the ${sourceType} Data Source to create. Must be slug-friendly.`,description: `An optional description for the ${sourceType} Data Source.`,projectId: `The ID of the project to create the ${sourceType} Data Source in.`,- connectionId: `The ID of the ${+ appConnectionId: `The ID of the ${APP_CONNECTION_NAME_MAP[SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP[type]]} Connection to use for this Data Source.`,isAutoScanEnabled: `Whether scans should be automatically performed when a ${autoScanDescription.verb} occurs to ${autoScanDescription.noun} associated with this Data Source.`,@@ -3850,7 +3850,7 @@ export const SecretScanningFindings = {UPDATE: {findingId: "The ID of the Secret Scanning Finding to update.",status: "The updated status of the specified Secret Scanning Finding.",- remarks: "Remarks pertaining to the status of this finding."+ triageComment: "A comment explaining the status given to this finding."}};diff --git a/backend/src/services/app-connection/app-connection-dal.ts b/backend/src/services/app-connection/app-connection-dal.tsindex 0135ad7fa0..51e3b59d0d 100644--- a/backend/src/services/app-connection/app-connection-dal.ts+++ b/backend/src/services/app-connection/app-connection-dal.ts@@ -136,7 +136,8 @@ export const appConnectionDALFactory = (db: TDbClient) => {);const dataSources = await (tx || db.replicaNode())(TableName.SecretScanningDataSource)- .where(`${TableName.SecretScanningDataSource}.connectionId`, connectionId)+ .where(`${TableName.SecretScanningDataSource}.appConnectionId`, connectionId)+ .whereNull(`${TableName.SecretScanningDataSource}.deletedAt`).join(TableName.Project, `${TableName.SecretScanningDataSource}.projectId`, `${TableName.Project}.id`).select(db.ref("name").withSchema(TableName.SecretScanningDataSource),diff --git a/backend/src/services/org-product-stats/org-product-stats-dal.ts b/backend/src/services/org-product-stats/org-product-stats-dal.tsindex ef2347c2a2..fdf9e15121 100644--- a/backend/src/services/org-product-stats/org-product-stats-dal.ts+++ b/backend/src/services/org-product-stats/org-product-stats-dal.ts@@ -80,6 +80,7 @@ export const orgProductStatsDALFactory = (db: TDbClient) => {.join(TableName.Project, `${TableName.SecretScanningDataSource}.projectId`, `${TableName.Project}.id`).where(`${TableName.Project}.orgId`, orgId).whereNull(`${TableName.Project}.deleteAfter`)+ .whereNull(`${TableName.SecretScanningDataSource}.deletedAt`).where(`${TableName.Project}.type`, ProjectType.SecretScanning).count(`${TableName.SecretScanningDataSource}.id as count`).first()) as { count: string } | undefined;@@ -160,12 +161,14 @@ export const orgProductStatsDALFactory = (db: TDbClient) => {const result = (await (tx || db.replicaNode())(TableName.SecretScanningResource).join(TableName.SecretScanningDataSource,- `${TableName.SecretScanningResource}.dataSourceId`,+ `${TableName.SecretScanningResource}.sourceId`,`${TableName.SecretScanningDataSource}.id`).join(TableName.Project, `${TableName.SecretScanningDataSource}.projectId`, `${TableName.Project}.id`).where(`${TableName.Project}.orgId`, orgId).whereNull(`${TableName.Project}.deleteAfter`)+ .whereNull(`${TableName.SecretScanningDataSource}.deletedAt`)+ .whereNull(`${TableName.SecretScanningResource}.deletedAt`).where(`${TableName.Project}.type`, ProjectType.SecretScanning).count(`${TableName.SecretScanningResource}.id as count`).first()) as { count: string } | undefined;diff --git a/frontend/src/components/secret-scanning/forms/SecretScanningDataSourceForm.tsx b/frontend/src/components/secret-scanning/forms/SecretScanningDataSourceForm.tsxindex 53b89c5c3e..ef7b7a3386 100644--- a/frontend/src/components/secret-scanning/forms/SecretScanningDataSourceForm.tsx+++ b/frontend/src/components/secret-scanning/forms/SecretScanningDataSourceForm.tsx@@ -70,7 +70,7 @@ export const SecretScanningDataSourceForm = ({}): createDataSource.mutateAsync({...formData,- connectionId: connection?.id,+ appConnectionId: connection?.id,projectId: currentProject.id});const source = await mutation;diff --git a/frontend/src/hooks/api/secretScanningV2/enums.ts b/frontend/src/hooks/api/secretScanningV2/enums.tsindex a4fd7bd34a..b2ce9f2524 100644--- a/frontend/src/hooks/api/secretScanningV2/enums.ts+++ b/frontend/src/hooks/api/secretScanningV2/enums.ts@@ -12,8 +12,13 @@ export enum SecretScanningScanStatus {}export enum SecretScanningScanType {- FullScan = "full-scan",- DiffScan = "diff-scan"+ Historical = "historical",+ Realtime = "realtime"+}++export enum SecretScanningScanTrigger {+ Manual = "manual",+ Push = "push"}export enum SecretScanningFindingStatus {@@ -33,3 +38,9 @@ export enum SecretScanningFindingSeverity {Medium = "medium",Low = "low"}++export enum SecretScanningFindingConfidence {+ High = "high",+ Medium = "medium",+ Low = "low"+}diff --git a/frontend/src/hooks/api/secretScanningV2/types/index.ts b/frontend/src/hooks/api/secretScanningV2/types/index.tsindex 2c45c9bcfd..262f272fa2 100644--- a/frontend/src/hooks/api/secretScanningV2/types/index.ts+++ b/frontend/src/hooks/api/secretScanningV2/types/index.ts@@ -6,10 +6,11 @@ import { DiscriminativePick } from "@app/types";import {SecretScanningDataSource,+ SecretScanningFindingConfidence,SecretScanningFindingSeverity,SecretScanningFindingStatus,- SecretScanningResource,SecretScanningScanStatus,+ SecretScanningScanTrigger,SecretScanningScanType} from "../enums";import { TBitbucketDataSource, TBitbucketDataSourceOption } from "./bitbucket-data-source";@@ -44,11 +45,11 @@ export type TSecretScanningDataSourceResponse = { dataSource: TSecretScanningDatexport type TCreateSecretScanningDataSourceDTO = DiscriminativePick<TSecretScanningDataSource,- "name" | "config" | "description" | "connectionId" | "type" | "isAutoScanEnabled" | "projectId"+ "name" | "config" | "description" | "appConnectionId" | "type" | "isAutoScanEnabled" | "projectId">;export type TUpdateSecretScanningDataSourceDTO = Partial<- Omit<TCreateSecretScanningDataSourceDTO, "type" | "connectionId" | "projectId">+ Omit<TCreateSecretScanningDataSourceDTO, "type" | "appConnectionId" | "projectId">> & {type: SecretScanningDataSource;dataSourceId: string;@@ -74,7 +75,7 @@ export type TTriggerSecretScanningDataSourceDTO = {export type TUpdateSecretScanningFinding = {findingId: string;status: SecretScanningFindingStatus;- remarks?: string | null;+ triageComment?: string | null;// required for query invalidationprojectId: string;};@@ -90,8 +91,7 @@ export type TGetSecretScanningDataSource = {export type TSecretScanningResourceWithDetails = {id: string;- dataSourceId: string;- type: SecretScanningResource;+ sourceId: string;externalId: string;name: string;createdAt: string;@@ -111,6 +111,10 @@ export type TSecretScanningScanWithDetails = {createdAt: string;resourceId: string;type: SecretScanningScanType;+ trigger: SecretScanningScanTrigger;+ triggeredByUserId?: string | null;+ startedAt?: string | null;+ completedAt?: string | null;status: SecretScanningScanStatus;statusMessage?: string | null;unresolvedFindings: number;@@ -137,14 +141,19 @@ export type TGetSecretScanningConfigResponse = {export type TSecretScanningFinding = {id: string;+ resourceId: string;+ resourceName: string;+ dataSourceId: string;dataSourceName: string;dataSourceType: SecretScanningDataSource;- resourceName: string;- resourceType: SecretScanningResource;- rule: string;+ ruleKey: string;severity: SecretScanningFindingSeverity;+ confidence?: SecretScanningFindingConfidence | null;status: SecretScanningFindingStatus;- remarks?: string;+ triageComment?: string | null;+ triagedByUserId?: string | null;+ triagedAt?: string | null;+ resolvedAt?: string | null;fingerprint: string;// TODO scott: this will need to be type differentiated once we add other scan typesdetails: {diff --git a/frontend/src/hooks/api/secretScanningV2/types/shared/secret-scanning-data-source-base.ts b/frontend/src/hooks/api/secretScanningV2/types/shared/secret-scanning-data-source-base.tsindex bcba0b02ac..6d861d82e2 100644--- a/frontend/src/hooks/api/secretScanningV2/types/shared/secret-scanning-data-source-base.ts+++ b/frontend/src/hooks/api/secretScanningV2/types/shared/secret-scanning-data-source-base.ts@@ -4,7 +4,7 @@ export type TSecretScanningDataSourceBase = {id: string;name: string;description?: string | null;- connectionId?: string | null;+ appConnectionId?: string | null;createdAt: string;updatedAt: string;projectId: string;diff --git a/frontend/src/pages/secret-scanning/SecretScanningDataSourceByIdPage/components/SecretScanningScanRow.tsx b/frontend/src/pages/secret-scanning/SecretScanningDataSourceByIdPage/components/SecretScanningScanRow.tsxindex 33fa33044b..0b472dc990 100644--- a/frontend/src/pages/secret-scanning/SecretScanningDataSourceByIdPage/components/SecretScanningScanRow.tsx+++ b/frontend/src/pages/secret-scanning/SecretScanningDataSourceByIdPage/components/SecretScanningScanRow.tsx@@ -84,7 +84,7 @@ export const SecretScanningScanRow = ({ scan }: Props) => {</div></Td><Td className="whitespace-nowrap">- {type === SecretScanningScanType.FullScan ? "Full Scan" : "Diff Scan"}+ {type === SecretScanningScanType.Historical ? "Full Scan" : "Diff Scan"}</Td><Td>{diff --git a/frontend/src/pages/secret-scanning/SecretScanningFindingsPage/components/SecretScanningFindingRow.tsx b/frontend/src/pages/secret-scanning/SecretScanningFindingsPage/components/SecretScanningFindingRow.tsxindex 37961fca1e..29e3c463c7 100644--- a/frontend/src/pages/secret-scanning/SecretScanningFindingsPage/components/SecretScanningFindingRow.tsx+++ b/frontend/src/pages/secret-scanning/SecretScanningFindingsPage/components/SecretScanningFindingRow.tsx@@ -50,10 +50,10 @@ export const SecretScanningFindingRow = ({dataSourceType,createdAt,dataSourceName,- rule,+ ruleKey,status,details,- remarks+ triageComment} = finding;const [isIdCopied, setIsIdCopied] = useToggle(false);@@ -123,9 +123,9 @@ export const SecretScanningFindingRow = ({<p className="truncate text-xs text-muted">{dataSourceName}</p></div></Td>- <Td className="whitespace-nowrap">{rule}</Td>+ <Td className="whitespace-nowrap">{ruleKey}</Td><Td className="whitespace-nowrap">- <Tooltip position="left" content={remarks}>+ <Tooltip position="left" content={triageComment}><Badge variant={SECRET_SCANNING_FINDING_STATUS_MAP[status].variant}><StatusIcon /><span className="capitalize">{status.replace("-", " ")}</span>@@ -226,9 +226,9 @@ export const SecretScanningFindingRow = ({</Button>)}</ProjectPermissionCan>- {Boolean(remarks) && (+ {Boolean(triageComment) && (<GenericFieldLabel className="col-span-3" label="Remarks">- {remarks}+ {triageComment}</GenericFieldLabel>)}</div>diff --git a/frontend/src/pages/secret-scanning/SecretScanningFindingsPage/components/SecretScanningFindingsTable.tsx b/frontend/src/pages/secret-scanning/SecretScanningFindingsPage/components/SecretScanningFindingsTable.tsxindex bbc2440dcf..87c83ff9ad 100644--- a/frontend/src/pages/secret-scanning/SecretScanningFindingsPage/components/SecretScanningFindingsTable.tsx+++ b/frontend/src/pages/secret-scanning/SecretScanningFindingsPage/components/SecretScanningFindingsTable.tsx@@ -114,7 +114,7 @@ export const SecretScanningFindingsTable = ({ findings }: Props) => {() =>findings.filter((finding) => {- const { rule, resourceName, dataSourceType, dataSourceName, status, scanId } = finding;+ const { ruleKey, resourceName, dataSourceType, dataSourceName, status, scanId } = finding;if (filters.dataSourceTypes.length && !filters.dataSourceTypes.includes(dataSourceType))return false;@@ -135,7 +135,7 @@ export const SecretScanningFindingsTable = ({ findings }: Props) => {.includes(searchValue) ||resourceName.toLowerCase().includes(searchValue) ||dataSourceName.toLowerCase().includes(searchValue) ||- rule.toLowerCase().includes(searchValue)+ ruleKey.toLowerCase().includes(searchValue));}).sort((a, b) => {@@ -148,7 +148,9 @@ export const SecretScanningFindingsTable = ({ findings }: Props) => {new Date(findingTwo.createdAt).getTime() - new Date(findingOne.createdAt).getTime());case FindingsOrderBy.Rule:- return findingOne.rule.toLowerCase().localeCompare(findingTwo.rule.toLowerCase());+ return findingOne.ruleKey+ .toLowerCase()+ .localeCompare(findingTwo.ruleKey.toLowerCase());case FindingsOrderBy.Status:return (getStatusOrderValue(findingOne.status) - getStatusOrderValue(findingTwo.status)diff --git a/frontend/src/pages/secret-scanning/SecretScanningFindingsPage/components/SecretScanningUpdateFindingModal.tsx b/frontend/src/pages/secret-scanning/SecretScanningFindingsPage/components/SecretScanningUpdateFindingModal.tsxindex 7a2677c2e4..66cd6b3a60 100644--- a/frontend/src/pages/secret-scanning/SecretScanningFindingsPage/components/SecretScanningUpdateFindingModal.tsx+++ b/frontend/src/pages/secret-scanning/SecretScanningFindingsPage/components/SecretScanningUpdateFindingModal.tsx@@ -28,7 +28,7 @@ type Props = {};const FormSchema = z.object({- remarks: z.string().max(256, "Cannot exceed 256 characters").optional(),+ triageComment: z.string().max(1024, "Cannot exceed 1024 characters").optional(),status: z.nativeEnum(SecretScanningFindingStatus).optional()});@@ -48,7 +48,7 @@ const Content = ({ findings, onComplete }: ContentProps) => {resolver: zodResolver(FormSchema),defaultValues: {status: single ? findings[0].status : undefined,- remarks: single ? findings[0].remarks : undefined+ triageComment: single ? (findings[0].triageComment ?? undefined) : undefined}});@@ -117,10 +117,10 @@ const Content = ({ findings, onComplete }: ContentProps) => {/><Controllercontrol={control}- name="remarks"+ name="triageComment"render={({ field, fieldState: { error } }) => {return (- <FormControl label="Remarks" isError={Boolean(error)} errorText={error?.message}>+ <FormControl label="Comment" isError={Boolean(error)} errorText={error?.message}><TextArea className="h-40 resize-none!" {...field} /></FormControl>);@@ -155,7 +155,7 @@ export const SecretScanningUpdateFindingModal = ({<Modal isOpen={isOpen} onOpenChange={onOpenChange}><ModalContenttitle={`Update Finding${findings.length === 1 ? "" : "s"}`}- subTitle="Update the status or leave remarks"+ subTitle="Update the status or leave a comment"><Contentfindings={findings}