Skip to content

Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.

add migration and update db schemas

Matheus Nogueira42cf6b7a82b1
diff --git a/backend/src/db/migrations/20260929120000_reshape-secret-scanning-v2-tables.ts b/backend/src/db/migrations/20260929120000_reshape-secret-scanning-v2-tables.ts
new file mode 100644
index 0000000000..bdb9828f13
--- /dev/null
+++ b/backend/src/db/migrations/20260929120000_reshape-secret-scanning-v2-tables.ts
@@ -0,0 +1,305 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+const SOURCE_ACTIVE_NAME_UNIQUE = "secret_scanning_data_sources_project_id_name_active_unique";
+const RESOURCE_ACTIVE_EXTERNAL_ID_UNIQUE = "secret_scanning_resources_source_id_external_id_active_unique";
+const SCAN_STARTED_AT_LEGACY_INDEX = "secret_scanning_scans_scanning_started_at_index";
+const SCAN_STARTED_AT_INDEX = "secret_scanning_scans_started_at_index";
+const SCAN_TRIGGERED_BY_INDEX = "secret_scanning_scans_triggered_by_user_id_index";
+const FINDING_TRIAGED_BY_INDEX = "secret_scanning_findings_triaged_by_user_id_index";
+
+const renameConstraint = (knex: Knex, table: TableName, from: string, to: string) =>
+ knex.raw(`ALTER TABLE ?? RENAME CONSTRAINT ?? TO ??`, [table, from, to]);
+
+export async function up(knex: Knex): Promise<void> {
+ // Scan workers write to these tables continuously; fail the deploy fast rather than queue behind them.
+ await knex.raw("SET LOCAL lock_timeout = '10s'");
+
+ if (await knex.schema.hasColumn(TableName.SecretScanningDataSource, "connectionId")) {
+ await knex.schema.alterTable(TableName.SecretScanningDataSource, (t) => {
+ t.dropUnique(["projectId", "name"]);
+ });
+ await knex.schema.alterTable(TableName.SecretScanningDataSource, (t) => {
+ t.renameColumn("connectionId", "appConnectionId");
+ t.timestamp("deletedAt").nullable();
+ });
+ // Keeps knex's derived name in step with the column, so a later dropForeign(["appConnectionId"]) finds it.
+ await renameConstraint(
+ knex,
+ TableName.SecretScanningDataSource,
+ "secret_scanning_data_sources_connectionid_foreign",
+ "secret_scanning_data_sources_appconnectionid_foreign"
+ );
+ await knex.schema.alterTable(TableName.SecretScanningDataSource, (t) => {
+ t.index("appConnectionId");
+ });
+ await knex.raw(`CREATE UNIQUE INDEX ?? ON ?? ("projectId", "name") WHERE "deletedAt" IS NULL`, [
+ SOURCE_ACTIVE_NAME_UNIQUE,
+ TableName.SecretScanningDataSource
+ ]);
+ }
+
+ if (await knex.schema.hasColumn(TableName.SecretScanningResource, "dataSourceId")) {
+ await knex.schema.alterTable(TableName.SecretScanningResource, (t) => {
+ t.dropUnique(["dataSourceId", "externalId"]);
+ });
+ await knex.schema.alterTable(TableName.SecretScanningResource, (t) => {
+ t.renameColumn("dataSourceId", "sourceId");
+ t.dropColumn("type");
+ t.timestamp("deletedAt").nullable();
+ });
+ await renameConstraint(
+ knex,
+ TableName.SecretScanningResource,
+ "secret_scanning_resources_datasourceid_foreign",
+ "secret_scanning_resources_sourceid_foreign"
+ );
+ await knex.raw(`CREATE UNIQUE INDEX ?? ON ?? ("sourceId", "externalId") WHERE "deletedAt" IS NULL`, [
+ RESOURCE_ACTIVE_EXTERNAL_ID_UNIQUE,
+ TableName.SecretScanningResource
+ ]);
+ }
+
+ if (await knex.schema.hasColumn(TableName.SecretScanningScan, "scanningStartedAt")) {
+ await knex.schema.alterTable(TableName.SecretScanningScan, (t) => {
+ t.renameColumn("scanningStartedAt", "startedAt");
+ });
+ await knex.schema.alterTable(TableName.SecretScanningScan, (t) => {
+ t.string("trigger").nullable();
+ t.uuid("triggeredByUserId").nullable();
+ t.foreign("triggeredByUserId").references("id").inTable(TableName.Users).onDelete("SET NULL");
+ t.timestamp("completedAt").nullable();
+ t.index("resourceId");
+ });
+ await knex.raw(`ALTER INDEX IF EXISTS ?? RENAME TO ??`, [SCAN_STARTED_AT_LEGACY_INDEX, SCAN_STARTED_AT_INDEX]);
+ await knex.raw(`CREATE INDEX ?? ON ?? ("triggeredByUserId") WHERE "triggeredByUserId" IS NOT NULL`, [
+ SCAN_TRIGGERED_BY_INDEX,
+ TableName.SecretScanningScan
+ ]);
+
+ // Every existing full scan was started by someone creating a source or pressing scan, so they count as
+ // manual; diff scans only ever came from a push webhook. SET expressions all read the pre-update row.
+ await knex.raw(
+ `UPDATE ?? SET
+ type = CASE type WHEN 'full-scan' THEN 'historical' WHEN 'diff-scan' THEN 'realtime' ELSE type END,
+ "trigger" = CASE type WHEN 'diff-scan' THEN 'push' ELSE 'manual' END,
+ "createdAt" = COALESCE("createdAt", "startedAt", "progressUpdatedAt", NOW())`,
+ [TableName.SecretScanningScan]
+ );
+
+ await knex.schema.alterTable(TableName.SecretScanningScan, (t) => {
+ t.string("trigger").notNullable().alter();
+ t.timestamp("createdAt").notNullable().defaultTo(knex.fn.now()).alter();
+ });
+ }
+
+ if (await knex.schema.hasColumn(TableName.SecretScanningFinding, "projectId")) {
+ await knex.schema.alterTable(TableName.SecretScanningFinding, (t) => {
+ t.uuid("resourceId").nullable();
+ });
+
+ await knex.raw(`UPDATE ?? AS f SET "resourceId" = s."resourceId" FROM ?? AS s WHERE s.id = f."scanId"`, [
+ TableName.SecretScanningFinding,
+ TableName.SecretScanningScan
+ ]);
+
+ // Every finding is inserted with its scan, and a scan only disappears when its resource is deleted
+ // (SET NULL on scanId), so a finding with no resource here belongs to a resource that no longer exists.
+ await knex(TableName.SecretScanningFinding).whereNull("resourceId").delete();
+
+ await knex.schema.alterTable(TableName.SecretScanningFinding, (t) => {
+ t.dropUnique(["projectId", "fingerprint"]);
+ t.dropForeign(["scanId"]);
+ });
+
+ await knex.schema.alterTable(TableName.SecretScanningFinding, (t) => {
+ t.uuid("resourceId").notNullable().alter();
+ t.uuid("scanId").notNullable().alter();
+ });
+
+ await knex.schema.alterTable(TableName.SecretScanningFinding, (t) => {
+ t.foreign("resourceId").references("id").inTable(TableName.SecretScanningResource).onDelete("CASCADE");
+ t.unique(["resourceId", "fingerprint"]);
+ t.foreign("scanId").references("id").inTable(TableName.SecretScanningScan).onDelete("CASCADE");
+ t.index("scanId");
+
+ t.renameColumn("rule", "ruleKey");
+ t.renameColumn("remarks", "triageComment");
+ });
+
+ await knex.schema.alterTable(TableName.SecretScanningFinding, (t) => {
+ t.text("triageComment").nullable().alter();
+ t.string("confidence").nullable();
+ t.uuid("triagedByUserId").nullable();
+ t.foreign("triagedByUserId").references("id").inTable(TableName.Users).onDelete("SET NULL");
+ t.timestamp("triagedAt").nullable();
+ t.timestamp("resolvedAt").nullable();
+ t.binary("encryptedFinding").nullable();
+
+ t.dropColumn("dataSourceName");
+ t.dropColumn("dataSourceType");
+ t.dropColumn("resourceName");
+ t.dropColumn("resourceType");
+ t.dropColumn("projectId");
+ });
+
+ await knex.raw(`CREATE INDEX ?? ON ?? ("triagedByUserId") WHERE "triagedByUserId" IS NOT NULL`, [
+ FINDING_TRIAGED_BY_INDEX,
+ TableName.SecretScanningFinding
+ ]);
+ }
+}
+
+export async function down(knex: Knex): Promise<void> {
+ await knex.raw("SET LOCAL lock_timeout = '10s'");
+
+ if (await knex.schema.hasColumn(TableName.SecretScanningFinding, "resourceId")) {
+ await knex.schema.alterTable(TableName.SecretScanningFinding, (t) => {
+ t.string("projectId").nullable();
+ t.string("dataSourceName").nullable();
+ t.string("dataSourceType").nullable();
+ t.string("resourceName").nullable();
+ t.string("resourceType").nullable();
+ });
+
+ await knex.raw(
+ `UPDATE ?? AS f SET
+ "projectId" = src."projectId",
+ "dataSourceName" = src.name,
+ "dataSourceType" = src.type,
+ "resourceName" = r.name,
+ "resourceType" = CASE src.type WHEN 'gitlab' THEN 'project' ELSE 'repository' END
+ FROM ?? AS r
+ JOIN ?? AS src ON src.id = r."sourceId"
+ WHERE r.id = f."resourceId"`,
+ [TableName.SecretScanningFinding, TableName.SecretScanningResource, TableName.SecretScanningDataSource]
+ );
+
+ // The old key is per project, so the same fingerprint in two repos of one project collapses to its oldest row.
+ await knex.raw(
+ `DELETE FROM ?? AS f USING ?? AS older
+ WHERE older."projectId" = f."projectId"
+ AND older.fingerprint = f.fingerprint
+ AND (older."createdAt", older.id) < (f."createdAt", f.id)`,
+ [TableName.SecretScanningFinding, TableName.SecretScanningFinding]
+ );
+
+ await knex.raw(`UPDATE ?? SET "triageComment" = LEFT("triageComment", 255) WHERE LENGTH("triageComment") > 255`, [
+ TableName.SecretScanningFinding
+ ]);
+
+ await knex.raw(`DROP INDEX IF EXISTS ??`, [FINDING_TRIAGED_BY_INDEX]);
+
+ await knex.schema.alterTable(TableName.SecretScanningFinding, (t) => {
+ t.dropUnique(["resourceId", "fingerprint"]);
+ t.dropForeign(["scanId"]);
+ t.dropIndex(["scanId"]);
+ t.dropForeign(["resourceId"]);
+ t.dropForeign(["triagedByUserId"]);
+ });
+
+ await knex.schema.alterTable(TableName.SecretScanningFinding, (t) => {
+ t.dropColumn("resourceId");
+ t.dropColumn("confidence");
+ t.dropColumn("triagedByUserId");
+ t.dropColumn("triagedAt");
+ t.dropColumn("resolvedAt");
+ t.dropColumn("encryptedFinding");
+
+ t.renameColumn("ruleKey", "rule");
+ t.renameColumn("triageComment", "remarks");
+ });
+
+ await knex.schema.alterTable(TableName.SecretScanningFinding, (t) => {
+ t.string("remarks").nullable().alter();
+ t.uuid("scanId").nullable().alter();
+ t.string("projectId").notNullable().alter();
+ t.string("dataSourceName").notNullable().alter();
+ t.string("dataSourceType").notNullable().alter();
+ t.string("resourceName").notNullable().alter();
+ t.string("resourceType").notNullable().alter();
+ });
+
+ await knex.schema.alterTable(TableName.SecretScanningFinding, (t) => {
+ t.foreign("projectId").references("id").inTable(TableName.Project).onDelete("CASCADE");
+ t.foreign("scanId").references("id").inTable(TableName.SecretScanningScan).onDelete("SET NULL");
+ t.unique(["projectId", "fingerprint"]);
+ });
+ }
+
+ if (await knex.schema.hasColumn(TableName.SecretScanningScan, "startedAt")) {
+ await knex.raw(
+ `UPDATE ?? SET type = CASE type WHEN 'historical' THEN 'full-scan' WHEN 'realtime' THEN 'diff-scan' ELSE type END`,
+ [TableName.SecretScanningScan]
+ );
+
+ await knex.raw(`DROP INDEX IF EXISTS ??`, [SCAN_TRIGGERED_BY_INDEX]);
+ await knex.raw(`ALTER INDEX IF EXISTS ?? RENAME TO ??`, [SCAN_STARTED_AT_INDEX, SCAN_STARTED_AT_LEGACY_INDEX]);
+
+ await knex.schema.alterTable(TableName.SecretScanningScan, (t) => {
+ t.dropForeign(["triggeredByUserId"]);
+ t.dropIndex(["resourceId"]);
+ });
+ await knex.schema.alterTable(TableName.SecretScanningScan, (t) => {
+ t.dropColumn("trigger");
+ t.dropColumn("triggeredByUserId");
+ t.dropColumn("completedAt");
+ t.renameColumn("startedAt", "scanningStartedAt");
+ });
+ await knex.schema.alterTable(TableName.SecretScanningScan, (t) => {
+ t.timestamp("createdAt").nullable().defaultTo(knex.fn.now()).alter();
+ });
+ }
+
+ if (await knex.schema.hasColumn(TableName.SecretScanningResource, "sourceId")) {
+ // Pre-migration code has no notion of a deleted resource and would treat these as live again.
+ await knex(TableName.SecretScanningResource).whereNotNull("deletedAt").delete();
+
+ await knex.raw(`DROP INDEX IF EXISTS ??`, [RESOURCE_ACTIVE_EXTERNAL_ID_UNIQUE]);
+ await knex.schema.alterTable(TableName.SecretScanningResource, (t) => {
+ t.string("type").nullable();
+ });
+ await knex.raw(
+ `UPDATE ?? AS r SET type = CASE src.type WHEN 'gitlab' THEN 'project' ELSE 'repository' END
+ FROM ?? AS src WHERE src.id = r."sourceId"`,
+ [TableName.SecretScanningResource, TableName.SecretScanningDataSource]
+ );
+ await renameConstraint(
+ knex,
+ TableName.SecretScanningResource,
+ "secret_scanning_resources_sourceid_foreign",
+ "secret_scanning_resources_datasourceid_foreign"
+ );
+ await knex.schema.alterTable(TableName.SecretScanningResource, (t) => {
+ t.dropColumn("deletedAt");
+ t.renameColumn("sourceId", "dataSourceId");
+ });
+ await knex.schema.alterTable(TableName.SecretScanningResource, (t) => {
+ t.string("type").notNullable().alter();
+ t.unique(["dataSourceId", "externalId"]);
+ });
+ }
+
+ if (await knex.schema.hasColumn(TableName.SecretScanningDataSource, "appConnectionId")) {
+ await knex(TableName.SecretScanningDataSource).whereNotNull("deletedAt").delete();
+
+ await knex.raw(`DROP INDEX IF EXISTS ??`, [SOURCE_ACTIVE_NAME_UNIQUE]);
+ await knex.schema.alterTable(TableName.SecretScanningDataSource, (t) => {
+ t.dropIndex(["appConnectionId"]);
+ });
+ await renameConstraint(
+ knex,
+ TableName.SecretScanningDataSource,
+ "secret_scanning_data_sources_appconnectionid_foreign",
+ "secret_scanning_data_sources_connectionid_foreign"
+ );
+ await knex.schema.alterTable(TableName.SecretScanningDataSource, (t) => {
+ t.dropColumn("deletedAt");
+ t.renameColumn("appConnectionId", "connectionId");
+ });
+ await knex.schema.alterTable(TableName.SecretScanningDataSource, (t) => {
+ t.unique(["projectId", "name"]);
+ });
+ }
+}
diff --git a/backend/src/db/schemas/secret-scanning-data-sources.ts b/backend/src/db/schemas/secret-scanning-data-sources.ts
index d79b45e790..515683f3ae 100644
--- a/backend/src/db/schemas/secret-scanning-data-sources.ts
+++ b/backend/src/db/schemas/secret-scanning-data-sources.ts
@@ -17,12 +17,13 @@ export const SecretScanningDataSourcesSchema = z.object({
type: z.string(),
config: z.unknown(),
encryptedCredentials: zodBuffer.nullable().optional(),
- connectionId: z.string().uuid().nullable().optional(),
+ appConnectionId: z.string().uuid().nullable().optional(),
isAutoScanEnabled: z.boolean().default(true).nullable().optional(),
projectId: z.string(),
createdAt: z.date(),
updatedAt: z.date(),
- isDisconnected: z.boolean().default(false)
+ isDisconnected: z.boolean().default(false),
+ deletedAt: z.date().nullable().optional()
});
export type TSecretScanningDataSources = z.infer<typeof SecretScanningDataSourcesSchema>;
diff --git a/backend/src/db/schemas/secret-scanning-findings.ts b/backend/src/db/schemas/secret-scanning-findings.ts
index c36f229f83..5880e31d61 100644
--- a/backend/src/db/schemas/secret-scanning-findings.ts
+++ b/backend/src/db/schemas/secret-scanning-findings.ts
@@ -5,24 +5,27 @@
import { z } from "zod";
+import { zodBuffer } from "@app/lib/zod";
+
import { TImmutableDBKeys } from "./models";
export const SecretScanningFindingsSchema = z.object({
id: z.string().uuid(),
- dataSourceName: z.string(),
- dataSourceType: z.string(),
- resourceName: z.string(),
- resourceType: z.string(),
- rule: z.string(),
+ ruleKey: z.string(),
severity: z.string(),
status: z.string().default("unresolved"),
- remarks: z.string().nullable().optional(),
+ triageComment: z.string().nullable().optional(),
fingerprint: z.string(),
details: z.unknown(),
- projectId: z.string(),
- scanId: z.string().uuid().nullable().optional(),
+ scanId: z.string().uuid(),
createdAt: z.date(),
- updatedAt: z.date()
+ updatedAt: z.date(),
+ resourceId: z.string().uuid(),
+ confidence: z.string().nullable().optional(),
+ triagedByUserId: z.string().uuid().nullable().optional(),
+ triagedAt: z.date().nullable().optional(),
+ resolvedAt: z.date().nullable().optional(),
+ encryptedFinding: zodBuffer.nullable().optional()
});
export type TSecretScanningFindings = z.infer<typeof SecretScanningFindingsSchema>;
diff --git a/backend/src/db/schemas/secret-scanning-resources.ts b/backend/src/db/schemas/secret-scanning-resources.ts
index e791e1cd62..610bccb07a 100644
--- a/backend/src/db/schemas/secret-scanning-resources.ts
+++ b/backend/src/db/schemas/secret-scanning-resources.ts
@@ -11,10 +11,10 @@ export const SecretScanningResourcesSchema = z.object({
id: z.string().uuid(),
externalId: z.string(),
name: z.string(),
- type: z.string(),
- dataSourceId: z.string().uuid(),
+ sourceId: z.string().uuid(),
createdAt: z.date(),
- updatedAt: z.date()
+ updatedAt: z.date(),
+ deletedAt: z.date().nullable().optional()
});
export type TSecretScanningResources = z.infer<typeof SecretScanningResourcesSchema>;
diff --git a/backend/src/db/schemas/secret-scanning-scans.ts b/backend/src/db/schemas/secret-scanning-scans.ts
index 51fdc8dcdc..71f3770730 100644
--- a/backend/src/db/schemas/secret-scanning-scans.ts
+++ b/backend/src/db/schemas/secret-scanning-scans.ts
@@ -13,11 +13,14 @@ export const SecretScanningScansSchema = z.object({
statusMessage: z.string().nullable().optional(),
type: z.string(),
resourceId: z.string().uuid(),
- createdAt: z.date().nullable().optional(),
- scanningStartedAt: z.date().nullable().optional(),
+ createdAt: z.date(),
+ startedAt: z.date().nullable().optional(),
lastScannedCommit: z.string().nullable().optional(),
progressUpdatedAt: z.date().nullable().optional(),
- lastScannedCommitDigest: z.string().nullable().optional()
+ lastScannedCommitDigest: z.string().nullable().optional(),
+ trigger: z.string(),
+ triggeredByUserId: z.string().uuid().nullable().optional(),
+ completedAt: z.date().nullable().optional()
});
export type TSecretScanningScans = z.infer<typeof SecretScanningScansSchema>;