Skip to content

Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.

revamp db tables for secret scanning

Matheus Nogueirabce5e01fd184
diff --git a/backend/src/ee/routes/v2/secret-scanning-v2-routers/secret-scanning-v2-endpoints.ts b/backend/src/ee/routes/v2/secret-scanning-v2-routers/secret-scanning-v2-endpoints.ts
index 3e2e941513..18a16a2cda 100644
--- a/backend/src/ee/routes/v2/secret-scanning-v2-routers/secret-scanning-v2-endpoints.ts
+++ b/backend/src/ee/routes/v2/secret-scanning-v2-routers/secret-scanning-v2-endpoints.ts
@@ -1,12 +1,15 @@
import { z } from "zod";
-import { SecretScanningResourcesSchema, SecretScanningScansSchema } from "@app/db/schemas";
import { EventType } from "@app/ee/services/audit-log/audit-log-types";
import {
SecretScanningDataSource,
SecretScanningScanStatus
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import { SECRET_SCANNING_DATA_SOURCE_NAME_MAP } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-maps";
+import {
+ SecretScanningResourceSchema,
+ SecretScanningScanSchema
+} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-schemas";
import {
TSecretScanningDataSource,
TSecretScanningDataSourceInput
@@ -34,7 +37,7 @@ export const registerSecretScanningEndpoints = <
createSchema: z.ZodType<{
name: string;
projectId: string;
- connectionId?: string;
+ appConnectionId?: string;
config: Partial<I["config"]>;
description?: string | null;
isAutoScanEnabled?: boolean;
@@ -444,7 +447,7 @@ export const registerSecretScanningEndpoints = <
dataSourceId: z.string().uuid().describe(SecretScanningDataSources.LIST_RESOURCES(type).dataSourceId)
}),
response: {
- 200: z.object({ resources: SecretScanningResourcesSchema.array() })
+ 200: z.object({ resources: SecretScanningResourceSchema.array() })
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN, AuthMode.OAUTH]),
@@ -489,7 +492,7 @@ export const registerSecretScanningEndpoints = <
dataSourceId: z.string().uuid().describe(SecretScanningDataSources.LIST_SCANS(type).dataSourceId)
}),
response: {
- 200: z.object({ scans: SecretScanningScansSchema.array() })
+ 200: z.object({ scans: SecretScanningScanSchema.array() })
}
},
onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN, AuthMode.OAUTH]),
@@ -533,7 +536,7 @@ export const registerSecretScanningEndpoints = <
}),
response: {
200: z.object({
- resources: SecretScanningResourcesSchema.extend({
+ resources: SecretScanningResourceSchema.extend({
lastScannedAt: z.date().nullish(),
lastScanStatus: z.nativeEnum(SecretScanningScanStatus).nullish(),
lastScanStatusMessage: z.string().nullish(),
@@ -584,7 +587,7 @@ export const registerSecretScanningEndpoints = <
}),
response: {
200: z.object({
- scans: SecretScanningScansSchema.extend({
+ scans: SecretScanningScanSchema.extend({
unresolvedFindings: z.number(),
resolvedFindings: z.number(),
resourceName: z.string()
diff --git a/backend/src/ee/routes/v2/secret-scanning-v2-routers/secret-scanning-v2-router.ts b/backend/src/ee/routes/v2/secret-scanning-v2-routers/secret-scanning-v2-router.ts
index a880131773..6ad3ad6d5e 100644
--- a/backend/src/ee/routes/v2/secret-scanning-v2-routers/secret-scanning-v2-router.ts
+++ b/backend/src/ee/routes/v2/secret-scanning-v2-routers/secret-scanning-v2-router.ts
@@ -164,7 +164,7 @@ export const registerSecretScanningV2Router = async (server: FastifyZodProvider)
}),
body: z.object({
status: z.nativeEnum(SecretScanningFindingStatus).optional().describe(SecretScanningFindings.UPDATE.status),
- remarks: z.string().nullish().describe(SecretScanningFindings.UPDATE.remarks)
+ triageComment: z.string().trim().max(1024).nullish().describe(SecretScanningFindings.UPDATE.triageComment)
}),
response: {
200: z.object({ finding: SecretScanningFindingSchema })
@@ -225,7 +225,7 @@ export const registerSecretScanningV2Router = async (server: FastifyZodProvider)
.object({
findingId: z.string().trim().min(1, "Finding ID required").describe(SecretScanningFindings.UPDATE.findingId),
status: z.nativeEnum(SecretScanningFindingStatus).optional().describe(SecretScanningFindings.UPDATE.status),
- remarks: z.string().nullish().describe(SecretScanningFindings.UPDATE.remarks)
+ triageComment: z.string().trim().max(1024).nullish().describe(SecretScanningFindings.UPDATE.triageComment)
})
.array()
.max(500),
diff --git a/backend/src/ee/services/secret-scanning-v2/bitbucket/bitbucket-secret-scanning-factory.ts b/backend/src/ee/services/secret-scanning-v2/bitbucket/bitbucket-secret-scanning-factory.ts
index 8804a8de55..65f592caa0 100644
--- a/backend/src/ee/services/secret-scanning-v2/bitbucket/bitbucket-secret-scanning-factory.ts
+++ b/backend/src/ee/services/secret-scanning-v2/bitbucket/bitbucket-secret-scanning-factory.ts
@@ -3,10 +3,7 @@ import RE2 from "re2";
import { scanContentAndGetFindings } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-fns";
import { SecretMatch } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";
-import {
- SecretScanningFindingSeverity,
- SecretScanningResource
-} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
+import { SecretScanningFindingSeverity } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import {
assertProviderRepositorySizeWithinLimit,
cloneRepository,
@@ -157,8 +154,7 @@ export const BitbucketSecretScanningFactory = () => {
return filteredRepos.map(({ full_name, uuid }) => ({
name: full_name,
- externalId: uuid,
- type: SecretScanningResource.Repository
+ externalId: uuid
}));
};
@@ -213,8 +209,7 @@ export const BitbucketSecretScanningFactory = () => {
> = ({ repository }) => {
return {
name: repository.full_name,
- externalId: repository.uuid,
- type: SecretScanningResource.Repository
+ externalId: repository.uuid
};
};
@@ -326,7 +321,7 @@ export const BitbucketSecretScanningFactory = () => {
details: toFindingDetails(finding),
fingerprint: finding.Fingerprint,
severity: SecretScanningFindingSeverity.High,
- rule: finding.RuleID
+ ruleKey: finding.RuleID
}));
};
diff --git a/backend/src/ee/services/secret-scanning-v2/bitbucket/bitbucket-secret-scanning-schemas.ts b/backend/src/ee/services/secret-scanning-v2/bitbucket/bitbucket-secret-scanning-schemas.ts
index 36d036f154..75c1d9d05d 100644
--- a/backend/src/ee/services/secret-scanning-v2/bitbucket/bitbucket-secret-scanning-schemas.ts
+++ b/backend/src/ee/services/secret-scanning-v2/bitbucket/bitbucket-secret-scanning-schemas.ts
@@ -1,9 +1,6 @@
import { z } from "zod";
-import {
- SecretScanningDataSource,
- SecretScanningResource
-} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
+import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import {
BaseCreateSecretScanningDataSourceSchema,
BaseSecretScanningDataSourceSchema,
@@ -86,7 +83,6 @@ export const BitbucketDataSourceListItemSchema = z
);
export const BitbucketFindingSchema = BaseSecretScanningFindingSchema.extend({
- resourceType: z.literal(SecretScanningResource.Repository),
dataSourceType: z.literal(SecretScanningDataSource.Bitbucket),
details: GitRepositoryScanFindingDetailsSchema
});
diff --git a/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-factory.ts b/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-factory.ts
index 97aae68004..eaf1d48179 100644
--- a/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-factory.ts
+++ b/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-factory.ts
@@ -5,8 +5,7 @@ import { scanContentAndGetFindings } from "@app/ee/services/secret-scanning/secr
import { SecretMatch } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";
import {
SecretScanningDataSource,
- SecretScanningFindingSeverity,
- SecretScanningResource
+ SecretScanningFindingSeverity
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import {
assertProviderRepositorySizeWithinLimit,
@@ -95,8 +94,7 @@ export const GitHubSecretScanningFactory = () => {
return filteredRepos.map(({ id, full_name }) => ({
name: full_name,
- externalId: id.toString(),
- type: SecretScanningResource.Repository
+ externalId: id.toString()
}));
};
@@ -154,8 +152,7 @@ export const GitHubSecretScanningFactory = () => {
> = ({ repository }) => {
return {
name: repository.full_name,
- externalId: repository.id.toString(),
- type: SecretScanningResource.Repository
+ externalId: repository.id.toString()
};
};
@@ -241,7 +238,7 @@ export const GitHubSecretScanningFactory = () => {
details: toFindingDetails(finding),
fingerprint: finding.Fingerprint,
severity: SecretScanningFindingSeverity.High,
- rule: finding.RuleID
+ ruleKey: finding.RuleID
}));
};
diff --git a/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-schemas.ts b/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-schemas.ts
index e39830f220..fbe2215dce 100644
--- a/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-schemas.ts
+++ b/backend/src/ee/services/secret-scanning-v2/github/github-secret-scanning-schemas.ts
@@ -1,9 +1,6 @@
import { z } from "zod";
-import {
- SecretScanningDataSource,
- SecretScanningResource
-} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
+import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import {
BaseCreateSecretScanningDataSourceSchema,
BaseSecretScanningDataSourceSchema,
@@ -79,7 +76,6 @@ export const GitHubDataSourceListItemSchema = z
);
export const GitHubFindingSchema = BaseSecretScanningFindingSchema.extend({
- resourceType: z.literal(SecretScanningResource.Repository),
dataSourceType: z.literal(SecretScanningDataSource.GitHub),
details: GitRepositoryScanFindingDetailsSchema
});
diff --git a/backend/src/ee/services/secret-scanning-v2/gitlab/gitlab-secret-scanning-factory.ts b/backend/src/ee/services/secret-scanning-v2/gitlab/gitlab-secret-scanning-factory.ts
index 443e4a1879..b0257a8935 100644
--- a/backend/src/ee/services/secret-scanning-v2/gitlab/gitlab-secret-scanning-factory.ts
+++ b/backend/src/ee/services/secret-scanning-v2/gitlab/gitlab-secret-scanning-factory.ts
@@ -3,10 +3,7 @@ import { join } from "path";
import { scanContentAndGetFindings } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-fns";
import { SecretMatch } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";
-import {
- SecretScanningFindingSeverity,
- SecretScanningResource
-} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
+import { SecretScanningFindingSeverity } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import {
assertProviderRepositorySizeWithinLimit,
cloneRepository,
@@ -214,8 +211,7 @@ export const GitLabSecretScanningFactory = ({ appConnectionDAL, kmsService }: TS
return [
{
name: project.pathWithNamespace,
- externalId: project.id.toString(),
- type: SecretScanningResource.Project
+ externalId: project.id.toString()
}
];
}
@@ -237,8 +233,7 @@ export const GitLabSecretScanningFactory = ({ appConnectionDAL, kmsService }: TS
return filteredProjects.map(({ id, pathWithNamespace }) => ({
name: pathWithNamespace,
- externalId: id.toString(),
- type: SecretScanningResource.Project
+ externalId: id.toString()
}));
};
@@ -308,8 +303,7 @@ export const GitLabSecretScanningFactory = ({ appConnectionDAL, kmsService }: TS
> = ({ project }) => {
return {
name: project.path_with_namespace,
- externalId: project.id.toString(),
- type: SecretScanningResource.Project
+ externalId: project.id.toString()
};
};
@@ -378,7 +372,7 @@ export const GitLabSecretScanningFactory = ({ appConnectionDAL, kmsService }: TS
details: toFindingDetails(finding),
fingerprint: finding.Fingerprint,
severity: SecretScanningFindingSeverity.High,
- rule: finding.RuleID
+ ruleKey: finding.RuleID
}));
};
diff --git a/backend/src/ee/services/secret-scanning-v2/gitlab/gitlab-secret-scanning-schemas.ts b/backend/src/ee/services/secret-scanning-v2/gitlab/gitlab-secret-scanning-schemas.ts
index e70e63c11c..1c91037ab2 100644
--- a/backend/src/ee/services/secret-scanning-v2/gitlab/gitlab-secret-scanning-schemas.ts
+++ b/backend/src/ee/services/secret-scanning-v2/gitlab/gitlab-secret-scanning-schemas.ts
@@ -1,10 +1,7 @@
import { z } from "zod";
import { GitLabDataSourceScope } from "@app/ee/services/secret-scanning-v2/gitlab/gitlab-secret-scanning-enums";
-import {
- SecretScanningDataSource,
- SecretScanningResource
-} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
+import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import {
BaseCreateSecretScanningDataSourceSchema,
BaseSecretScanningDataSourceSchema,
@@ -90,7 +87,6 @@ export const GitLabDataSourceListItemSchema = z
);
export const GitLabFindingSchema = BaseSecretScanningFindingSchema.extend({
- resourceType: z.literal(SecretScanningResource.Project),
dataSourceType: z.literal(SecretScanningDataSource.GitLab),
details: GitRepositoryScanFindingDetailsSchema
});
diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-dal.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-dal.ts
index aadc6dc83e..34de7627c7 100644
--- a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-dal.ts
+++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-dal.ts
@@ -6,7 +6,9 @@ import {
SecretScanningScansSchema,
TableName,
TSecretScanningDataSources,
- TSecretScanningFindings
+ TSecretScanningFindings,
+ TSecretScanningResources,
+ TSecretScanningResourcesInsert
} from "@app/db/schemas";
import {
SecretScanningFindingStatus,
@@ -41,9 +43,10 @@ const baseSecretScanningDataSourceQuery = ({
const query = (tx || db.replicaNode())(TableName.SecretScanningDataSource)
.join(
TableName.AppConnection,
- `${TableName.SecretScanningDataSource}.connectionId`,
+ `${TableName.SecretScanningDataSource}.appConnectionId`,
`${TableName.AppConnection}.id`
)
+ .whereNull(`${TableName.SecretScanningDataSource}.deletedAt`)
.select(selectAllTableCols(TableName.SecretScanningDataSource))
.select(
// entire connection
@@ -82,7 +85,7 @@ const expandSecretScanningDataSource = <
const {
connectionApp,
connectionName,
- connectionId,
+ appConnectionId,
connectionOrgId,
connectionIsAutoRotationEnabled,
connectionEncryptedCredentials,
@@ -100,11 +103,11 @@ const expandSecretScanningDataSource = <
return {
...el,
- connectionId,
- connection: connectionId
+ appConnectionId,
+ connection: appConnectionId
? {
app: connectionApp,
- id: connectionId,
+ id: appConnectionId,
name: connectionName,
orgId: connectionOrgId,
encryptedCredentials: connectionEncryptedCredentials,
@@ -200,6 +203,8 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {
return expandSecretScanningDataSource(dataSource);
};
+ // Soft delete: the source's findings stay in the database for audit, but every read path filters
+ // deleted sources out, and the partial unique index frees the name for reuse.
const deleteDataSourceById = async (dataSourceId: string, tx?: Knex) => {
const dataSource = (await baseSecretScanningDataSourceQuery({
filter: { id: dataSourceId },
@@ -207,7 +212,7 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {
tx
}).first())!;
- await dataSourceOrm.deleteById(dataSourceId, tx);
+ await dataSourceOrm.updateById(dataSourceId, { deletedAt: new Date() }, tx);
return expandSecretScanningDataSource(dataSource);
};
@@ -229,11 +234,11 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {
// TODO (scott): this query will probably need to be optimized
const dataSources = await baseSecretScanningDataSourceQuery({ filter, db, tx })
- .leftJoin(
- TableName.SecretScanningResource,
- `${TableName.SecretScanningResource}.dataSourceId`,
- `${TableName.SecretScanningDataSource}.id`
- )
+ .leftJoin(TableName.SecretScanningResource, (qb) => {
+ void qb
+ .on(`${TableName.SecretScanningResource}.sourceId`, `${TableName.SecretScanningDataSource}.id`)
+ .andOnNull(`${TableName.SecretScanningResource}.deletedAt`);
+ })
.leftJoin(
TableName.SecretScanningScan,
`${TableName.SecretScanningScan}.resourceId`,
@@ -315,6 +320,7 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {
if (filter)
void qb.where(buildFindFilter(prependTableNameToFindFilter(TableName.SecretScanningResource, filter)));
})
+ .whereNull(`${TableName.SecretScanningResource}.deletedAt`)
.leftJoin(
TableName.SecretScanningScan,
`${TableName.SecretScanningScan}.resourceId`,
@@ -400,7 +406,8 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {
`${TableName.SecretScanningResource}.id`,
`${TableName.SecretScanningScan}.resourceId`
)
- .where(`${TableName.SecretScanningResource}.dataSourceId`, dataSourceId)
+ .where(`${TableName.SecretScanningResource}.sourceId`, dataSourceId)
+ .whereNull(`${TableName.SecretScanningResource}.deletedAt`)
.leftJoin(
TableName.SecretScanningFinding,
`${TableName.SecretScanningFinding}.scanId`,
@@ -466,7 +473,7 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {
// time so they're reaped too.
.whereRaw(`COALESCE(??, ??, ??) < ?`, [
`${TableName.SecretScanningScan}.progressUpdatedAt`,
- `${TableName.SecretScanningScan}.scanningStartedAt`,
+ `${TableName.SecretScanningScan}.startedAt`,
`${TableName.SecretScanningScan}.createdAt`,
startedBefore
])
@@ -478,8 +485,7 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {
.select(selectAllTableCols(TableName.SecretScanningScan))
.select(
db.ref("name").withSchema(TableName.SecretScanningResource).as("resourceName"),
- db.ref("type").withSchema(TableName.SecretScanningResource).as("resourceType"),
- db.ref("dataSourceId").withSchema(TableName.SecretScanningResource)
+ db.ref("sourceId").withSchema(TableName.SecretScanningResource).as("dataSourceId")
)
.orderBy(`${TableName.SecretScanningScan}.createdAt`, "asc")
.limit(limit);
@@ -498,8 +504,8 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {
`${TableName.SecretScanningResource}.id`,
`${TableName.SecretScanningScan}.resourceId`
)
- .where(`${TableName.SecretScanningResource}.dataSourceId`, dataSourceId)
-
+ .where(`${TableName.SecretScanningResource}.sourceId`, dataSourceId)
+ .whereNull(`${TableName.SecretScanningResource}.deletedAt`)
.select(selectAllTableCols(TableName.SecretScanningScan));
return scans;
@@ -508,6 +514,103 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {
}
};
+ // The unique index on (sourceId, externalId) is partial (active rows only), and an ON CONFLICT
+ // target has to repeat that predicate to match it, which ormify's upsert cannot express.
+ const upsertResources = async (data: TSecretScanningResourcesInsert[], tx?: Knex) => {
+ if (!data.length) return [];
+
+ try {
+ const { sql, bindings } = (tx || db)(TableName.SecretScanningResource).insert(data).toSQL();
+
+ const result = (await (tx || db).raw(
+ `${sql} ON CONFLICT ("sourceId", "externalId") WHERE "deletedAt" IS NULL DO UPDATE SET "name" = EXCLUDED."name" RETURNING *`,
+ bindings
+ )) as { rows: TSecretScanningResources[] };
+
+ return result.rows;
+ } catch (error) {
+ throw new DatabaseError({ error, name: "Upsert - Secret Scanning Resource" });
+ }
+ };
+
+ const baseFindingQuery = (tx?: Knex) =>
+ (tx || db.replicaNode())(TableName.SecretScanningFinding)
+ .join(
+ TableName.SecretScanningResource,
+ `${TableName.SecretScanningResource}.id`,
+ `${TableName.SecretScanningFinding}.resourceId`
+ )
+ .join(
+ TableName.SecretScanningDataSource,
+ `${TableName.SecretScanningDataSource}.id`,
+ `${TableName.SecretScanningResource}.sourceId`
+ )
+ .whereNull(`${TableName.SecretScanningResource}.deletedAt`)
+ .whereNull(`${TableName.SecretScanningDataSource}.deletedAt`);
+
+ const selectFindingWithDetails = <T extends Knex.QueryBuilder>(query: T) =>
+ query
+ .select(selectAllTableCols(TableName.SecretScanningFinding))
+ .select(
+ db.ref("projectId").withSchema(TableName.SecretScanningDataSource),
+ db.ref("id").withSchema(TableName.SecretScanningDataSource).as("dataSourceId"),
+ db.ref("name").withSchema(TableName.SecretScanningDataSource).as("dataSourceName"),
+ db.ref("type").withSchema(TableName.SecretScanningDataSource).as("dataSourceType"),
+ db.ref("name").withSchema(TableName.SecretScanningResource).as("resourceName")
+ );
+
+ type TSecretScanningFindingWithDetails = TSecretScanningFindings & {
+ projectId: string;
+ dataSourceId: string;
+ dataSourceName: string;
+ dataSourceType: string;
+ resourceName: string;
+ };
+
+ const findFindingsByProjectId = async (projectId: string, tx?: Knex) => {
+ try {
+ const findings = (await selectFindingWithDetails(
+ baseFindingQuery(tx).where(`${TableName.SecretScanningDataSource}.projectId`, projectId)
+ )) as TSecretScanningFindingWithDetails[];
+
+ return findings;
+ } catch (error) {
+ throw new DatabaseError({ error, name: "Find By Project ID - Secret Scanning Finding" });
+ }
+ };
+
+ const findFindingByIdWithDetails = async (findingId: string, tx?: Knex) => {
+ try {
+ const finding = (await selectFindingWithDetails(
+ baseFindingQuery(tx).where(`${TableName.SecretScanningFinding}.id`, findingId)
+ ).first()) as TSecretScanningFindingWithDetails | undefined;
+
+ return finding;
+ } catch (error) {
+ throw new DatabaseError({ error, name: "Find By ID - Secret Scanning Finding" });
+ }
+ };
+
+ const countFindingsByProjectId = async (
+ projectId: string,
+ filter: { status?: SecretScanningFindingStatus } = {},
+ tx?: Knex
+ ) => {
+ try {
+ const query = baseFindingQuery(tx).where(`${TableName.SecretScanningDataSource}.projectId`, projectId);
+
+ if (filter.status) void query.where(`${TableName.SecretScanningFinding}.status`, filter.status);
+
+ const result = (await query.count(`${TableName.SecretScanningFinding}.id as count`).first()) as
+ | { count: string }
+ | undefined;
+
+ return Number(result?.count ?? 0);
+ } catch (error) {
+ throw new DatabaseError({ error, name: "Count By Project ID - Secret Scanning Finding" });
+ }
+ };
+
/**
* Counts the findings this scan was the first to discover, which is the number the scan's own
* detail view shows. It is read back from the database rather than accumulated in the handler
@@ -541,6 +644,7 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {
},
resources: {
...resourceOrm,
+ upsert: upsertResources,
findWithDetails: findResourcesWithDetails
},
scans: {
@@ -552,7 +656,10 @@ export const secretScanningV2DALFactory = (db: TDbClient) => {
findings: {
...findingOrm,
upsert: upsertFindings,
- countByScanId: countFindingsByScanId
+ countByScanId: countFindingsByScanId,
+ findByProjectId: findFindingsByProjectId,
+ findByIdWithDetails: findFindingByIdWithDetails,
+ countByProjectId: countFindingsByProjectId
},
configs: configOrm
};
diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-enums.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-enums.ts
index a4fd7bd34a..b2ce9f2524 100644
--- a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-enums.ts
+++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-enums.ts
@@ -12,8 +12,13 @@ export enum SecretScanningScanStatus {
}
export enum SecretScanningScanType {
- FullScan = "full-scan",
- DiffScan = "diff-scan"
+ Historical = "historical",
+ Realtime = "realtime"
+}
+
+export enum SecretScanningScanTrigger {
+ Manual = "manual",
+ Push = "push"
}
export enum SecretScanningFindingStatus {
@@ -33,3 +38,9 @@ export enum SecretScanningFindingSeverity {
Medium = "medium",
Low = "low"
}
+
+export enum SecretScanningFindingConfidence {
+ High = "high",
+ Medium = "medium",
+ Low = "low"
+}
diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.ts
index 966117c0da..0d3cd0184b 100644
--- a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.ts
+++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-fns.ts
@@ -360,7 +360,7 @@ export const scanGitRepositoryAndGetFindings = async (
details: toFindingDetails(finding),
fingerprint: `${finding.Fingerprint}:${finding.StartColumn}`,
severity: SecretScanningFindingSeverity.High,
- rule: finding.RuleID
+ ruleKey: finding.RuleID
}));
};
diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-maps.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-maps.ts
index fd29e0b74f..c514946dea 100644
--- a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-maps.ts
+++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-maps.ts
@@ -1,4 +1,7 @@
-import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
+import {
+ SecretScanningDataSource,
+ SecretScanningResource
+} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import { AppConnection } from "@app/services/app-connection/app-connection-enums";
export const SECRET_SCANNING_DATA_SOURCE_NAME_MAP: Record<SecretScanningDataSource, string> = {
@@ -18,3 +21,9 @@ export const AUTO_SYNC_DESCRIPTION_HELPER: Record<SecretScanningDataSource, { ve
[SecretScanningDataSource.Bitbucket]: { verb: "push", noun: "repositories" },
[SecretScanningDataSource.GitLab]: { verb: "push", noun: "projects" }
};
+
+export const SECRET_SCANNING_DATA_SOURCE_RESOURCE_TYPE_MAP: Record<SecretScanningDataSource, SecretScanningResource> = {
+ [SecretScanningDataSource.GitHub]: SecretScanningResource.Repository,
+ [SecretScanningDataSource.Bitbucket]: SecretScanningResource.Repository,
+ [SecretScanningDataSource.GitLab]: SecretScanningResource.Project
+};
diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts
index e1f6e1f450..73cc3e528b 100644
--- a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts
+++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-queue.ts
@@ -36,9 +36,11 @@ import {
SecretScanningDataSource,
SecretScanningResource,
SecretScanningScanStatus,
+ SecretScanningScanTrigger,
SecretScanningScanType
} from "./secret-scanning-v2-enums";
import { SECRET_SCANNING_FACTORY_MAP } from "./secret-scanning-v2-factory";
+import { SECRET_SCANNING_DATA_SOURCE_RESOURCE_TYPE_MAP } from "./secret-scanning-v2-maps";
import {
TFindingsPayload,
TQueueSecretScanningDataSourceFullScan,
@@ -139,7 +141,7 @@ export const secretScanningV2QueueServiceFactory = ({
},
{
status: SecretScanningScanStatus.Scanning,
- scanningStartedAt: new Date(),
+ startedAt: new Date(),
...(trackProgress ? { progressUpdatedAt: new Date() } : {})
}
);
@@ -160,7 +162,8 @@ export const secretScanningV2QueueServiceFactory = ({
},
{
status: SecretScanningScanStatus.Failed,
- statusMessage
+ statusMessage,
+ completedAt: new Date()
}
);
@@ -169,7 +172,7 @@ export const secretScanningV2QueueServiceFactory = ({
const queueDataSourceFullScan = async (
dataSource: TSecretScanningDataSourceWithConnection,
- resourceExternalId?: string
+ { resourceExternalId, triggeredByUserId }: { resourceExternalId?: string; triggeredByUserId?: string } = {}
) => {
try {
const { type } = dataSource;
@@ -198,15 +201,14 @@ export const secretScanningV2QueueServiceFactory = ({
const resources = await secretScanningV2DAL.resources.upsert(
filteredRawResources.map((rawResource) => ({
...rawResource,
- dataSourceId: dataSource.id
+ sourceId: dataSource.id
})),
- ["externalId", "dataSourceId"],
tx
);
const inFlightScans = await secretScanningV2DAL.scans.find(
{
- type: SecretScanningScanType.FullScan,
+ type: SecretScanningScanType.Historical,
$in: {
resourceId: resources.map((resource) => resource.id),
status: [SecretScanningScanStatus.Queued, SecretScanningScanStatus.Scanning]
@@ -223,7 +225,9 @@ export const secretScanningV2QueueServiceFactory = ({
const scans = await secretScanningV2DAL.scans.insertMany(
resourcesToScan.map((resource) => ({
resourceId: resource.id,
- type: SecretScanningScanType.FullScan
+ type: SecretScanningScanType.Historical,
+ trigger: SecretScanningScanTrigger.Manual,
+ triggeredByUserId: triggeredByUserId ?? null
})),
tx
);
@@ -273,7 +277,7 @@ export const secretScanningV2QueueServiceFactory = ({
// Logged before any work so a `ps` on a saturated worker can be tied back to a scan ID.
logger.info(
- `secretScanningV2Queue: Full Scan Started ${logDetails} [scanType=${SecretScanningScanType.FullScan}] [tempFolder=${tempFolder}]`
+ `secretScanningV2Queue: Full Scan Started ${logDetails} [scanType=${SecretScanningScanType.Historical}] [tempFolder=${tempFolder}]`
);
const dataSource = await secretScanningV2DAL.dataSources.findById(dataSourceId);
@@ -288,6 +292,8 @@ export const secretScanningV2QueueServiceFactory = ({
if (!scan) throw new Error(`Scan with ID "${scanId}" not found`);
+ const resourceType = SECRET_SCANNING_DATA_SOURCE_RESOURCE_TYPE_MAP[dataSource.type as SecretScanningDataSource];
+
let holdsLease = false;
try {
@@ -305,7 +311,7 @@ export const secretScanningV2QueueServiceFactory = ({
if (!started) {
logger.warn(
- `secretScanningV2Queue: Full Scan skipped, scan was already closed out ${logDetails} [scanType=${SecretScanningScanType.FullScan}]`
+ `secretScanningV2Queue: Full Scan skipped, scan was already closed out ${logDetails} [scanType=${SecretScanningScanType.Historical}]`
);
return;
}
@@ -360,16 +366,13 @@ export const secretScanningV2QueueServiceFactory = ({
await secretScanningV2DAL.findings.upsert(
batchFindings.map((finding) => ({
...finding,
- projectId: dataSource.projectId,
- dataSourceName: dataSource.name,
- dataSourceType: dataSource.type,
- resourceName: resource.name,
- resourceType: resource.type,
+ resourceId,
scanId
})),
- ["projectId", "fingerprint"],
+ ["resourceId", "fingerprint"],
tx,
- ["resourceName", "dataSourceName"]
+ // a no-op merge, so rows already found by an earlier scan are returned with their original scanId
+ ["ruleKey"]
);
}
@@ -397,7 +400,7 @@ export const secretScanningV2QueueServiceFactory = ({
let stillOwned = true;
- switch (resource.type) {
+ switch (resourceType) {
case SecretScanningResource.Repository:
case SecretScanningResource.Project: {
const repoSizeMb = await assertClonedRepositoryWithinSizeLimit(resource.name, scanPath);
@@ -462,7 +465,8 @@ export const secretScanningV2QueueServiceFactory = ({
{ id: scanId, status: SecretScanningScanStatus.Scanning },
{
status: SecretScanningScanStatus.Completed,
- statusMessage: null
+ statusMessage: null,
+ completedAt: new Date()
}
)
: [];
@@ -506,10 +510,10 @@ export const secretScanningV2QueueServiceFactory = ({
dataSourceId: dataSource.id,
dataSourceType: dataSource.type,
resourceId: resource.id,
- resourceType: resource.type,
+ resourceType,
scanId,
scanStatus: SecretScanningScanStatus.Completed,
- scanType: SecretScanningScanType.FullScan,
+ scanType: SecretScanningScanType.Historical,
numberOfSecretsDetected: findingsCount
}
}
@@ -549,10 +553,10 @@ export const secretScanningV2QueueServiceFactory = ({
dataSourceId: dataSource.id,
dataSourceType: dataSource.type,
resourceId: resource.id,
- resourceType: resource.type,
+ resourceType,
scanId,
scanStatus: SecretScanningScanStatus.Failed,
- scanType: SecretScanningScanType.FullScan
+ scanType: SecretScanningScanType.Historical
}
}
});
@@ -588,17 +592,17 @@ export const secretScanningV2QueueServiceFactory = ({
[
{
...resourcePayload,
- dataSourceId
+ sourceId: dataSourceId
}
],
- ["externalId", "dataSourceId"],
tx
);
const scan = await secretScanningV2DAL.scans.create(
{
resourceId: resource.id,
- type: SecretScanningScanType.DiffScan
+ type: SecretScanningScanType.Realtime,
+ trigger: SecretScanningScanTrigger.Push
},
tx
);
@@ -658,10 +662,12 @@ export const secretScanningV2QueueServiceFactory = ({
appConnectionDAL
});
+ const resourceType = SECRET_SCANNING_DATA_SOURCE_RESOURCE_TYPE_MAP[dataSource.type as SecretScanningDataSource];
+
const tempFolder = await createTempFolder();
logger.info(
- `secretScanningV2Queue: Diff Scan Started ${logDetails} [scanType=${SecretScanningScanType.DiffScan}] [tempFolder=${tempFolder}]`
+ `secretScanningV2Queue: Diff Scan Started ${logDetails} [scanType=${SecretScanningScanType.Realtime}] [tempFolder=${tempFolder}]`
);
try {
@@ -669,7 +675,7 @@ export const secretScanningV2QueueServiceFactory = ({
if (!started) {
logger.warn(
- `secretScanningV2Queue: Diff Scan skipped, scan was already closed out ${logDetails} [scanType=${SecretScanningScanType.DiffScan}]`
+ `secretScanningV2Queue: Diff Scan skipped, scan was already closed out ${logDetails} [scanType=${SecretScanningScanType.Realtime}]`
);
return;
}
@@ -705,16 +711,12 @@ export const secretScanningV2QueueServiceFactory = ({
findings = await secretScanningV2DAL.findings.upsert(
findingsPayload.map((finding) => ({
...finding,
- projectId: dataSource.projectId,
- dataSourceName: dataSource.name,
- dataSourceType: dataSource.type,
- resourceName: resource.name,
- resourceType: resource.type,
+ resourceId,
scanId
})),
- ["projectId", "fingerprint"],
+ ["resourceId", "fingerprint"],
tx,
- ["resourceName", "dataSourceName"]
+ ["ruleKey"]
);
}
@@ -722,7 +724,8 @@ export const secretScanningV2QueueServiceFactory = ({
const completedScans = await secretScanningV2DAL.scans.update(
{ id: scanId, status: SecretScanningScanStatus.Scanning },
{
- status: SecretScanningScanStatus.Completed
+ status: SecretScanningScanStatus.Completed,
+ completedAt: new Date()
},
tx
);
@@ -770,10 +773,10 @@ export const secretScanningV2QueueServiceFactory = ({
dataSourceId: dataSource.id,
dataSourceType: dataSource.type,
resourceId,
- resourceType: resource.type,
+ resourceType,
scanId,
scanStatus: SecretScanningScanStatus.Completed,
- scanType: SecretScanningScanType.DiffScan,
+ scanType: SecretScanningScanType.Realtime,
numberOfSecretsDetected: findingsPayload.length
}
}
@@ -813,10 +816,10 @@ export const secretScanningV2QueueServiceFactory = ({
dataSourceId: dataSource.id,
dataSourceType: dataSource.type,
resourceId: resource.id,
- resourceType: resource.type,
+ resourceType,
scanId,
scanStatus: SecretScanningScanStatus.Failed,
- scanType: SecretScanningScanType.DiffScan
+ scanType: SecretScanningScanType.Realtime
}
}
});
@@ -929,7 +932,7 @@ export const secretScanningV2QueueServiceFactory = ({
// so the row it set to `scanning` stays that way forever and the customer sees a scan permanently
// in progress.
const failStuckScan = async (scan: Awaited<ReturnType<typeof secretScanningV2DAL.scans.findStuck>>[number]) => {
- const logDetails = `[scanId=${scan.id}] [resourceId=${scan.resourceId}] [dataSourceId=${scan.dataSourceId}] [scanningStartedAt=${scan.scanningStartedAt?.toISOString()}]`;
+ const logDetails = `[scanId=${scan.id}] [resourceId=${scan.resourceId}] [dataSourceId=${scan.dataSourceId}] [startedAt=${scan.startedAt?.toISOString()}]`;
try {
// Guarded on the status we read: if the worker did finish between the read and this write, the
@@ -938,7 +941,8 @@ export const secretScanningV2QueueServiceFactory = ({
{ id: scan.id, status: SecretScanningScanStatus.Scanning },
{
status: SecretScanningScanStatus.Failed,
- statusMessage: STUCK_SCAN_STATUS_MESSAGE
+ statusMessage: STUCK_SCAN_STATUS_MESSAGE,
+ completedAt: new Date()
}
);
@@ -974,7 +978,7 @@ export const secretScanningV2QueueServiceFactory = ({
dataSourceId: dataSource.id,
dataSourceType: dataSource.type,
resourceId: scan.resourceId,
- resourceType: scan.resourceType,
+ resourceType: SECRET_SCANNING_DATA_SOURCE_RESOURCE_TYPE_MAP[dataSource.type],
scanId: scan.id,
scanStatus: SecretScanningScanStatus.Failed,
scanType: scan.type as SecretScanningScanType
diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-schemas.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-schemas.ts
index 730427a18f..09636ac5a7 100644
--- a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-schemas.ts
+++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-schemas.ts
@@ -1,6 +1,11 @@
import { z } from "zod";
-import { SecretScanningDataSourcesSchema, SecretScanningFindingsSchema } from "@app/db/schemas";
+import {
+ SecretScanningDataSourcesSchema,
+ SecretScanningFindingsSchema,
+ SecretScanningResourcesSchema,
+ SecretScanningScansSchema
+} from "@app/db/schemas";
import { SecretScanningDataSource } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-enums";
import { SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP } from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-maps";
import { SecretScanningDataSources } from "@app/lib/api-docs";
@@ -18,11 +23,14 @@ export const BaseSecretScanningDataSourceSchema = ({
SecretScanningDataSourcesSchema.omit({
// unique to provider
type: true,
- connectionId: true,
- config: true
+ appConnectionId: true,
+ config: true,
+ // internal
+ encryptedCredentials: true,
+ deletedAt: true
}).extend({
type: z.literal(type),
- connectionId: isConnectionRequired ? z.string().uuid() : z.null(),
+ appConnectionId: isConnectionRequired ? z.string().uuid() : z.null(),
connection: isConnectionRequired
? z.object({
app: z.literal(SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP[type]),
@@ -49,8 +57,8 @@ export const BaseCreateSecretScanningDataSourceSchema = ({
.max(256, "Description cannot exceed 256 characters")
.nullish()
.describe(SecretScanningDataSources.CREATE(type).description),
- connectionId: isConnectionRequired
- ? z.string().uuid().describe(SecretScanningDataSources.CREATE(type).connectionId)
+ appConnectionId: isConnectionRequired
+ ? z.string().uuid().describe(SecretScanningDataSources.CREATE(type).appConnectionId)
: z.undefined(),
isAutoScanEnabled: z
.boolean()
@@ -92,7 +100,21 @@ export const GitRepositoryScanFindingDetailsSchema = z.object({
});
export const BaseSecretScanningFindingSchema = SecretScanningFindingsSchema.omit({
- dataSourceType: true,
- resourceType: true,
- details: true
+ details: true,
+ encryptedFinding: true
+}).extend({
+ projectId: z.string(),
+ dataSourceId: z.string().uuid(),
+ dataSourceName: z.string(),
+ resourceName: z.string()
+});
+
+export const SecretScanningResourceSchema = SecretScanningResourcesSchema.omit({
+ deletedAt: true
+});
+
+export const SecretScanningScanSchema = SecretScanningScansSchema.omit({
+ lastScannedCommit: true,
+ lastScannedCommitDigest: true,
+ progressUpdatedAt: true
});
diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-service.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-service.ts
index 36a425d7b2..9fadc17213 100644
--- a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-service.ts
+++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-service.ts
@@ -47,6 +47,7 @@ import { DatabaseErrorCode } from "@app/lib/error-codes";
import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors";
import { logger } from "@app/lib/logger";
import { OrgServiceActor } from "@app/lib/types";
+import { ActorType } from "@app/services/auth/auth-type";
import { TAppConnectionDALFactory } from "@app/services/app-connection/app-connection-dal";
import { decryptAppConnection } from "@app/services/app-connection/app-connection-fns";
import { TAppConnectionServiceFactory } from "@app/services/app-connection/app-connection-service";
@@ -251,11 +252,11 @@ export const secretScanningV2ServiceFactory = ({
);
let connection: TAppConnection | null = null;
- if (payload.connectionId) {
+ if (payload.appConnectionId) {
// validates permission to connect and app is valid for data source
connection = await appConnectionService.validateAppConnectionUsageById(
SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP[payload.type],
- { connectionId: payload.connectionId, projectId: payload.projectId },
+ { connectionId: payload.appConnectionId, projectId: payload.projectId },
actor
);
}
@@ -312,10 +313,13 @@ export const secretScanningV2ServiceFactory = ({
if (payload.isAutoScanEnabled) {
try {
- await secretScanningV2Queue.queueDataSourceFullScan({
- ...createdDataSource,
- connection
- } as TSecretScanningDataSourceWithConnection);
+ await secretScanningV2Queue.queueDataSourceFullScan(
+ {
+ ...createdDataSource,
+ connection
+ } as TSecretScanningDataSourceWithConnection,
+ { triggeredByUserId: actor.type === ActorType.USER ? actor.id : undefined }
+ );
} catch (error) {
// Deliberately non-fatal — a scan that can't be queued shouldn't block creating the data source.
logger.error(
@@ -376,11 +380,11 @@ export const secretScanningV2ServiceFactory = ({
});
let connection: TAppConnection | null = null;
- if (dataSource.connectionId) {
+ if (dataSource.appConnectionId) {
// validates permission to connect and app is valid for data source
connection = await appConnectionService.validateAppConnectionUsageById(
SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP[dataSource.type],
- { connectionId: dataSource.connectionId, projectId: dataSource.projectId },
+ { connectionId: dataSource.appConnectionId, projectId: dataSource.projectId },
actor
);
}
@@ -536,7 +540,11 @@ export const secretScanningV2ServiceFactory = ({
let resourceExternalId: string | undefined;
if (resourceId) {
- const resource = await secretScanningV2DAL.resources.findOne({ id: resourceId, dataSourceId });
+ const resource = await secretScanningV2DAL.resources.findOne({
+ id: resourceId,
+ sourceId: dataSourceId,
+ deletedAt: null
+ });
if (!resource) {
throw new NotFoundError({
message: `Could not find Secret Scanning Resource with ID "${resourceId}" for Data Source with ID "${dataSourceId}"`
@@ -550,7 +558,7 @@ export const secretScanningV2ServiceFactory = ({
...dataSource,
connection
} as TSecretScanningDataSourceWithConnection,
- resourceExternalId
+ { resourceExternalId, triggeredByUserId: actor.type === ActorType.USER ? actor.id : undefined }
);
return dataSource as TSecretScanningDataSource;
@@ -595,7 +603,8 @@ export const secretScanningV2ServiceFactory = ({
});
const resources = await secretScanningV2DAL.resources.find({
- dataSourceId
+ sourceId: dataSourceId,
+ deletedAt: null
});
return { resources, projectId: dataSource.projectId };
@@ -682,7 +691,7 @@ export const secretScanningV2ServiceFactory = ({
message: `Secret Scanning Data Source with ID "${dataSourceId}" is not configured for ${SECRET_SCANNING_DATA_SOURCE_NAME_MAP[type]}`
});
- const resources = await secretScanningV2DAL.resources.findWithDetails({ dataSourceId });
+ const resources = await secretScanningV2DAL.resources.findWithDetails({ sourceId: dataSourceId });
return { resources: resources as TSecretScanningResourceWithDetails[], projectId: dataSource.projectId };
};
@@ -753,15 +762,9 @@ export const secretScanningV2ServiceFactory = ({
ProjectPermissionSub.SecretScanningFindings
);
- const [finding] = await secretScanningV2DAL.findings.find(
- {
- projectId,
- status: SecretScanningFindingStatus.Unresolved
- },
- { count: true }
- );
-
- return Number(finding?.count ?? 0);
+ return secretScanningV2DAL.findings.countByProjectId(projectId, {
+ status: SecretScanningFindingStatus.Unresolved
+ });
};
const listSecretScanningFindingsByProjectId = async (projectId: string, actor: OrgServiceActor) => {
@@ -787,15 +790,13 @@ export const secretScanningV2ServiceFactory = ({
ProjectPermissionSub.SecretScanningFindings
);
- const findings = await secretScanningV2DAL.findings.find({
- projectId
- });
+ const findings = await secretScanningV2DAL.findings.findByProjectId(projectId);
return findings as TSecretScanningFinding[];
};
const updateSecretScanningFindingById = async (
- { findingId, remarks, status }: TUpdateSecretScanningFindingDTO,
+ { findingId, triageComment, status }: TUpdateSecretScanningFindingDTO,
actor: OrgServiceActor
) => {
const plan = await licenseService.getPlan(actor.orgId);
@@ -806,7 +807,7 @@ export const secretScanningV2ServiceFactory = ({
"Failed to access Secret Scanning Findings due to plan restriction. Upgrade plan to enable Secret Scanning."
});
- const finding = await secretScanningV2DAL.findings.findById(findingId);
+ const finding = await secretScanningV2DAL.findings.findByIdWithDetails(findingId);
if (!finding)
throw new NotFoundError({
@@ -827,9 +828,25 @@ export const secretScanningV2ServiceFactory = ({
ProjectPermissionSub.SecretScanningFindings
);
- const updatedFinding = await secretScanningV2DAL.findings.updateById(findingId, {
- remarks,
- status
+ const isStatusChange = status !== undefined && status !== finding.status;
+ const now = new Date();
+
+ const updatedFinding = await secretScanningV2DAL.findings.transaction(async (tx) => {
+ await secretScanningV2DAL.findings.updateById(
+ findingId,
+ {
+ triageComment,
+ status,
+ ...(isStatusChange || triageComment !== undefined
+ ? { triagedAt: now, triagedByUserId: actor.type === ActorType.USER ? actor.id : null }
+ : {}),
+ ...(isStatusChange ? { resolvedAt: status === SecretScanningFindingStatus.Resolved ? now : null } : {})
+ },
+ tx
+ );
+
+ // read back in the same transaction: a replica may not have the write yet
+ return secretScanningV2DAL.findings.findByIdWithDetails(findingId, tx);
});
return { finding: updatedFinding as TSecretScanningFinding, projectId: finding.projectId };
diff --git a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-types.ts b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-types.ts
index d885b05f5f..33818d52cf 100644
--- a/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-types.ts
+++ b/backend/src/ee/services/secret-scanning-v2/secret-scanning-v2-types.ts
@@ -103,14 +103,14 @@ export type TCreateSecretScanningDataSourceDTO = Pick<
TSecretScanningDataSource,
"description" | "name" | "projectId"
> & {
- connectionId?: string;
+ appConnectionId?: string;
type: SecretScanningDataSource;
isAutoScanEnabled?: boolean;
config: Partial<TSecretScanningDataSourceInput["config"]>;
};
export type TUpdateSecretScanningDataSourceDTO = Partial<
- Omit<TCreateSecretScanningDataSourceDTO, "projectId" | "connectionId">
+ Omit<TCreateSecretScanningDataSourceDTO, "projectId" | "appConnectionId">
> & {
dataSourceId: string;
type: SecretScanningDataSource;
@@ -168,11 +168,11 @@ export type TCloneRepository = {
export type TSecretScanningFactoryListRawResources<T extends TSecretScanningDataSourceWithConnection> = (
dataSource: T
-) => Promise<Pick<TSecretScanningResources, "externalId" | "name" | "type">[]>;
+) => Promise<Pick<TSecretScanningResources, "externalId" | "name">[]>;
export type TSecretScanningFactoryGetDiffScanResourcePayload<
P extends TQueueSecretScanningResourceDiffScan["payload"]
-> = (payload: P) => Pick<TSecretScanningResources, "externalId" | "name" | "type">;
+> = (payload: P) => Pick<TSecretScanningResources, "externalId" | "name">;
export type TSecretScanningFactoryGetFullScanPath<T extends TSecretScanningDataSourceWithConnection> = (parameters: {
dataSource: T;
@@ -239,12 +239,15 @@ export type TSecretScanningFactory<
getDiffScanFindingsPayload: TSecretScanningFactoryGetDiffScanFindingsPayload<T, P>;
};
-export type TFindingsPayload = Pick<TSecretScanningFindingsInsert, "details" | "fingerprint" | "severity" | "rule">[];
+export type TFindingsPayload = Pick<
+ TSecretScanningFindingsInsert,
+ "details" | "fingerprint" | "severity" | "ruleKey"
+>[];
export type TGetFindingsPayload = Promise<TFindingsPayload>;
export type TUpdateSecretScanningFindingDTO = {
status?: SecretScanningFindingStatus;
- remarks?: string | null;
+ triageComment?: string | null;
findingId: string;
};
diff --git a/backend/src/lib/api-docs/constants.ts b/backend/src/lib/api-docs/constants.ts
index a698fe1802..da3b614ab3 100644
--- a/backend/src/lib/api-docs/constants.ts
+++ b/backend/src/lib/api-docs/constants.ts
@@ -3792,7 +3792,7 @@ export const SecretScanningDataSources = {
name: `The name of the ${sourceType} Data Source to create. Must be slug-friendly.`,
description: `An optional description for the ${sourceType} Data Source.`,
projectId: `The ID of the project to create the ${sourceType} Data Source in.`,
- connectionId: `The ID of the ${
+ appConnectionId: `The ID of the ${
APP_CONNECTION_NAME_MAP[SECRET_SCANNING_DATA_SOURCE_CONNECTION_MAP[type]]
} Connection to use for this Data Source.`,
isAutoScanEnabled: `Whether scans should be automatically performed when a ${autoScanDescription.verb} occurs to ${autoScanDescription.noun} associated with this Data Source.`,
@@ -3850,7 +3850,7 @@ export const SecretScanningFindings = {
UPDATE: {
findingId: "The ID of the Secret Scanning Finding to update.",
status: "The updated status of the specified Secret Scanning Finding.",
- remarks: "Remarks pertaining to the status of this finding."
+ triageComment: "A comment explaining the status given to this finding."
}
};
diff --git a/backend/src/services/app-connection/app-connection-dal.ts b/backend/src/services/app-connection/app-connection-dal.ts
index 0135ad7fa0..51e3b59d0d 100644
--- a/backend/src/services/app-connection/app-connection-dal.ts
+++ b/backend/src/services/app-connection/app-connection-dal.ts
@@ -136,7 +136,8 @@ export const appConnectionDALFactory = (db: TDbClient) => {
);
const dataSources = await (tx || db.replicaNode())(TableName.SecretScanningDataSource)
- .where(`${TableName.SecretScanningDataSource}.connectionId`, connectionId)
+ .where(`${TableName.SecretScanningDataSource}.appConnectionId`, connectionId)
+ .whereNull(`${TableName.SecretScanningDataSource}.deletedAt`)
.join(TableName.Project, `${TableName.SecretScanningDataSource}.projectId`, `${TableName.Project}.id`)
.select(
db.ref("name").withSchema(TableName.SecretScanningDataSource),
diff --git a/backend/src/services/org-product-stats/org-product-stats-dal.ts b/backend/src/services/org-product-stats/org-product-stats-dal.ts
index ef2347c2a2..fdf9e15121 100644
--- a/backend/src/services/org-product-stats/org-product-stats-dal.ts
+++ b/backend/src/services/org-product-stats/org-product-stats-dal.ts
@@ -80,6 +80,7 @@ export const orgProductStatsDALFactory = (db: TDbClient) => {
.join(TableName.Project, `${TableName.SecretScanningDataSource}.projectId`, `${TableName.Project}.id`)
.where(`${TableName.Project}.orgId`, orgId)
.whereNull(`${TableName.Project}.deleteAfter`)
+ .whereNull(`${TableName.SecretScanningDataSource}.deletedAt`)
.where(`${TableName.Project}.type`, ProjectType.SecretScanning)
.count(`${TableName.SecretScanningDataSource}.id as count`)
.first()) as { count: string } | undefined;
@@ -160,12 +161,14 @@ export const orgProductStatsDALFactory = (db: TDbClient) => {
const result = (await (tx || db.replicaNode())(TableName.SecretScanningResource)
.join(
TableName.SecretScanningDataSource,
- `${TableName.SecretScanningResource}.dataSourceId`,
+ `${TableName.SecretScanningResource}.sourceId`,
`${TableName.SecretScanningDataSource}.id`
)
.join(TableName.Project, `${TableName.SecretScanningDataSource}.projectId`, `${TableName.Project}.id`)
.where(`${TableName.Project}.orgId`, orgId)
.whereNull(`${TableName.Project}.deleteAfter`)
+ .whereNull(`${TableName.SecretScanningDataSource}.deletedAt`)
+ .whereNull(`${TableName.SecretScanningResource}.deletedAt`)
.where(`${TableName.Project}.type`, ProjectType.SecretScanning)
.count(`${TableName.SecretScanningResource}.id as count`)
.first()) as { count: string } | undefined;
diff --git a/frontend/src/components/secret-scanning/forms/SecretScanningDataSourceForm.tsx b/frontend/src/components/secret-scanning/forms/SecretScanningDataSourceForm.tsx
index 53b89c5c3e..ef7b7a3386 100644
--- a/frontend/src/components/secret-scanning/forms/SecretScanningDataSourceForm.tsx
+++ b/frontend/src/components/secret-scanning/forms/SecretScanningDataSourceForm.tsx
@@ -70,7 +70,7 @@ export const SecretScanningDataSourceForm = ({
})
: createDataSource.mutateAsync({
...formData,
- connectionId: connection?.id,
+ appConnectionId: connection?.id,
projectId: currentProject.id
});
const source = await mutation;
diff --git a/frontend/src/hooks/api/secretScanningV2/enums.ts b/frontend/src/hooks/api/secretScanningV2/enums.ts
index a4fd7bd34a..b2ce9f2524 100644
--- a/frontend/src/hooks/api/secretScanningV2/enums.ts
+++ b/frontend/src/hooks/api/secretScanningV2/enums.ts
@@ -12,8 +12,13 @@ export enum SecretScanningScanStatus {
}
export enum SecretScanningScanType {
- FullScan = "full-scan",
- DiffScan = "diff-scan"
+ Historical = "historical",
+ Realtime = "realtime"
+}
+
+export enum SecretScanningScanTrigger {
+ Manual = "manual",
+ Push = "push"
}
export enum SecretScanningFindingStatus {
@@ -33,3 +38,9 @@ export enum SecretScanningFindingSeverity {
Medium = "medium",
Low = "low"
}
+
+export enum SecretScanningFindingConfidence {
+ High = "high",
+ Medium = "medium",
+ Low = "low"
+}
diff --git a/frontend/src/hooks/api/secretScanningV2/types/index.ts b/frontend/src/hooks/api/secretScanningV2/types/index.ts
index 2c45c9bcfd..262f272fa2 100644
--- a/frontend/src/hooks/api/secretScanningV2/types/index.ts
+++ b/frontend/src/hooks/api/secretScanningV2/types/index.ts
@@ -6,10 +6,11 @@ import { DiscriminativePick } from "@app/types";
import {
SecretScanningDataSource,
+ SecretScanningFindingConfidence,
SecretScanningFindingSeverity,
SecretScanningFindingStatus,
- SecretScanningResource,
SecretScanningScanStatus,
+ SecretScanningScanTrigger,
SecretScanningScanType
} from "../enums";
import { TBitbucketDataSource, TBitbucketDataSourceOption } from "./bitbucket-data-source";
@@ -44,11 +45,11 @@ export type TSecretScanningDataSourceResponse = { dataSource: TSecretScanningDat
export type TCreateSecretScanningDataSourceDTO = DiscriminativePick<
TSecretScanningDataSource,
- "name" | "config" | "description" | "connectionId" | "type" | "isAutoScanEnabled" | "projectId"
+ "name" | "config" | "description" | "appConnectionId" | "type" | "isAutoScanEnabled" | "projectId"
>;
export type TUpdateSecretScanningDataSourceDTO = Partial<
- Omit<TCreateSecretScanningDataSourceDTO, "type" | "connectionId" | "projectId">
+ Omit<TCreateSecretScanningDataSourceDTO, "type" | "appConnectionId" | "projectId">
> & {
type: SecretScanningDataSource;
dataSourceId: string;
@@ -74,7 +75,7 @@ export type TTriggerSecretScanningDataSourceDTO = {
export type TUpdateSecretScanningFinding = {
findingId: string;
status: SecretScanningFindingStatus;
- remarks?: string | null;
+ triageComment?: string | null;
// required for query invalidation
projectId: string;
};
@@ -90,8 +91,7 @@ export type TGetSecretScanningDataSource = {
export type TSecretScanningResourceWithDetails = {
id: string;
- dataSourceId: string;
- type: SecretScanningResource;
+ sourceId: string;
externalId: string;
name: string;
createdAt: string;
@@ -111,6 +111,10 @@ export type TSecretScanningScanWithDetails = {
createdAt: string;
resourceId: string;
type: SecretScanningScanType;
+ trigger: SecretScanningScanTrigger;
+ triggeredByUserId?: string | null;
+ startedAt?: string | null;
+ completedAt?: string | null;
status: SecretScanningScanStatus;
statusMessage?: string | null;
unresolvedFindings: number;
@@ -137,14 +141,19 @@ export type TGetSecretScanningConfigResponse = {
export type TSecretScanningFinding = {
id: string;
+ resourceId: string;
+ resourceName: string;
+ dataSourceId: string;
dataSourceName: string;
dataSourceType: SecretScanningDataSource;
- resourceName: string;
- resourceType: SecretScanningResource;
- rule: string;
+ ruleKey: string;
severity: SecretScanningFindingSeverity;
+ confidence?: SecretScanningFindingConfidence | null;
status: SecretScanningFindingStatus;
- remarks?: string;
+ triageComment?: string | null;
+ triagedByUserId?: string | null;
+ triagedAt?: string | null;
+ resolvedAt?: string | null;
fingerprint: string;
// TODO scott: this will need to be type differentiated once we add other scan types
details: {
diff --git a/frontend/src/hooks/api/secretScanningV2/types/shared/secret-scanning-data-source-base.ts b/frontend/src/hooks/api/secretScanningV2/types/shared/secret-scanning-data-source-base.ts
index bcba0b02ac..6d861d82e2 100644
--- a/frontend/src/hooks/api/secretScanningV2/types/shared/secret-scanning-data-source-base.ts
+++ b/frontend/src/hooks/api/secretScanningV2/types/shared/secret-scanning-data-source-base.ts
@@ -4,7 +4,7 @@ export type TSecretScanningDataSourceBase = {
id: string;
name: string;
description?: string | null;
- connectionId?: string | null;
+ appConnectionId?: string | null;
createdAt: string;
updatedAt: string;
projectId: string;
diff --git a/frontend/src/pages/secret-scanning/SecretScanningDataSourceByIdPage/components/SecretScanningScanRow.tsx b/frontend/src/pages/secret-scanning/SecretScanningDataSourceByIdPage/components/SecretScanningScanRow.tsx
index 33fa33044b..0b472dc990 100644
--- a/frontend/src/pages/secret-scanning/SecretScanningDataSourceByIdPage/components/SecretScanningScanRow.tsx
+++ b/frontend/src/pages/secret-scanning/SecretScanningDataSourceByIdPage/components/SecretScanningScanRow.tsx
@@ -84,7 +84,7 @@ export const SecretScanningScanRow = ({ scan }: Props) => {
</div>
</Td>
<Td className="whitespace-nowrap">
- {type === SecretScanningScanType.FullScan ? "Full Scan" : "Diff Scan"}
+ {type === SecretScanningScanType.Historical ? "Full Scan" : "Diff Scan"}
</Td>
<Td>
{
diff --git a/frontend/src/pages/secret-scanning/SecretScanningFindingsPage/components/SecretScanningFindingRow.tsx b/frontend/src/pages/secret-scanning/SecretScanningFindingsPage/components/SecretScanningFindingRow.tsx
index 37961fca1e..29e3c463c7 100644
--- a/frontend/src/pages/secret-scanning/SecretScanningFindingsPage/components/SecretScanningFindingRow.tsx
+++ b/frontend/src/pages/secret-scanning/SecretScanningFindingsPage/components/SecretScanningFindingRow.tsx
@@ -50,10 +50,10 @@ export const SecretScanningFindingRow = ({
dataSourceType,
createdAt,
dataSourceName,
- rule,
+ ruleKey,
status,
details,
- remarks
+ triageComment
} = finding;
const [isIdCopied, setIsIdCopied] = useToggle(false);
@@ -123,9 +123,9 @@ export const SecretScanningFindingRow = ({
<p className="truncate text-xs text-muted">{dataSourceName}</p>
</div>
</Td>
- <Td className="whitespace-nowrap">{rule}</Td>
+ <Td className="whitespace-nowrap">{ruleKey}</Td>
<Td className="whitespace-nowrap">
- <Tooltip position="left" content={remarks}>
+ <Tooltip position="left" content={triageComment}>
<Badge variant={SECRET_SCANNING_FINDING_STATUS_MAP[status].variant}>
<StatusIcon />
<span className="capitalize">{status.replace("-", " ")}</span>
@@ -226,9 +226,9 @@ export const SecretScanningFindingRow = ({
</Button>
)}
</ProjectPermissionCan>
- {Boolean(remarks) && (
+ {Boolean(triageComment) && (
<GenericFieldLabel className="col-span-3" label="Remarks">
- {remarks}
+ {triageComment}
</GenericFieldLabel>
)}
</div>
diff --git a/frontend/src/pages/secret-scanning/SecretScanningFindingsPage/components/SecretScanningFindingsTable.tsx b/frontend/src/pages/secret-scanning/SecretScanningFindingsPage/components/SecretScanningFindingsTable.tsx
index bbc2440dcf..87c83ff9ad 100644
--- a/frontend/src/pages/secret-scanning/SecretScanningFindingsPage/components/SecretScanningFindingsTable.tsx
+++ b/frontend/src/pages/secret-scanning/SecretScanningFindingsPage/components/SecretScanningFindingsTable.tsx
@@ -114,7 +114,7 @@ export const SecretScanningFindingsTable = ({ findings }: Props) => {
() =>
findings
.filter((finding) => {
- const { rule, resourceName, dataSourceType, dataSourceName, status, scanId } = finding;
+ const { ruleKey, resourceName, dataSourceType, dataSourceName, status, scanId } = finding;
if (filters.dataSourceTypes.length && !filters.dataSourceTypes.includes(dataSourceType))
return false;
@@ -135,7 +135,7 @@ export const SecretScanningFindingsTable = ({ findings }: Props) => {
.includes(searchValue) ||
resourceName.toLowerCase().includes(searchValue) ||
dataSourceName.toLowerCase().includes(searchValue) ||
- rule.toLowerCase().includes(searchValue)
+ ruleKey.toLowerCase().includes(searchValue)
);
})
.sort((a, b) => {
@@ -148,7 +148,9 @@ export const SecretScanningFindingsTable = ({ findings }: Props) => {
new Date(findingTwo.createdAt).getTime() - new Date(findingOne.createdAt).getTime()
);
case FindingsOrderBy.Rule:
- return findingOne.rule.toLowerCase().localeCompare(findingTwo.rule.toLowerCase());
+ return findingOne.ruleKey
+ .toLowerCase()
+ .localeCompare(findingTwo.ruleKey.toLowerCase());
case FindingsOrderBy.Status:
return (
getStatusOrderValue(findingOne.status) - getStatusOrderValue(findingTwo.status)
diff --git a/frontend/src/pages/secret-scanning/SecretScanningFindingsPage/components/SecretScanningUpdateFindingModal.tsx b/frontend/src/pages/secret-scanning/SecretScanningFindingsPage/components/SecretScanningUpdateFindingModal.tsx
index 7a2677c2e4..66cd6b3a60 100644
--- a/frontend/src/pages/secret-scanning/SecretScanningFindingsPage/components/SecretScanningUpdateFindingModal.tsx
+++ b/frontend/src/pages/secret-scanning/SecretScanningFindingsPage/components/SecretScanningUpdateFindingModal.tsx
@@ -28,7 +28,7 @@ type Props = {
};
const FormSchema = z.object({
- remarks: z.string().max(256, "Cannot exceed 256 characters").optional(),
+ triageComment: z.string().max(1024, "Cannot exceed 1024 characters").optional(),
status: z.nativeEnum(SecretScanningFindingStatus).optional()
});
@@ -48,7 +48,7 @@ const Content = ({ findings, onComplete }: ContentProps) => {
resolver: zodResolver(FormSchema),
defaultValues: {
status: single ? findings[0].status : undefined,
- remarks: single ? findings[0].remarks : undefined
+ triageComment: single ? (findings[0].triageComment ?? undefined) : undefined
}
});
@@ -117,10 +117,10 @@ const Content = ({ findings, onComplete }: ContentProps) => {
/>
<Controller
control={control}
- name="remarks"
+ name="triageComment"
render={({ field, fieldState: { error } }) => {
return (
- <FormControl label="Remarks" isError={Boolean(error)} errorText={error?.message}>
+ <FormControl label="Comment" isError={Boolean(error)} errorText={error?.message}>
<TextArea className="h-40 resize-none!" {...field} />
</FormControl>
);
@@ -155,7 +155,7 @@ export const SecretScanningUpdateFindingModal = ({
<Modal isOpen={isOpen} onOpenChange={onOpenChange}>
<ModalContent
title={`Update Finding${findings.length === 1 ? "" : "s"}`}
- subTitle="Update the status or leave remarks"
+ subTitle="Update the status or leave a comment"
>
<Content
findings={findings}