Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.
TypeScript acmeagentsai-agentscertificate-managementdevsecopsenvironment-variableskubernetesopen-sourcepampkipostgresprivate-caprivileged-access-managementsecret-managementsecret-managersecretssecrets-managementsecuritysecurity-toolsvault
add migration and update db schemas
Matheus Nogueira42cf6b7a82b1
diff --git a/backend/src/db/migrations/20260929120000_reshape-secret-scanning-v2-tables.ts b/backend/src/db/migrations/20260929120000_reshape-secret-scanning-v2-tables.tsnew file mode 100644index 0000000000..bdb9828f13--- /dev/null+++ b/backend/src/db/migrations/20260929120000_reshape-secret-scanning-v2-tables.ts@@ -0,0 +1,305 @@+import { Knex } from "knex";++import { TableName } from "../schemas";++const SOURCE_ACTIVE_NAME_UNIQUE = "secret_scanning_data_sources_project_id_name_active_unique";+const RESOURCE_ACTIVE_EXTERNAL_ID_UNIQUE = "secret_scanning_resources_source_id_external_id_active_unique";+const SCAN_STARTED_AT_LEGACY_INDEX = "secret_scanning_scans_scanning_started_at_index";+const SCAN_STARTED_AT_INDEX = "secret_scanning_scans_started_at_index";+const SCAN_TRIGGERED_BY_INDEX = "secret_scanning_scans_triggered_by_user_id_index";+const FINDING_TRIAGED_BY_INDEX = "secret_scanning_findings_triaged_by_user_id_index";++const renameConstraint = (knex: Knex, table: TableName, from: string, to: string) =>+ knex.raw(`ALTER TABLE ?? RENAME CONSTRAINT ?? TO ??`, [table, from, to]);++export async function up(knex: Knex): Promise<void> {+ // Scan workers write to these tables continuously; fail the deploy fast rather than queue behind them.+ await knex.raw("SET LOCAL lock_timeout = '10s'");++ if (await knex.schema.hasColumn(TableName.SecretScanningDataSource, "connectionId")) {+ await knex.schema.alterTable(TableName.SecretScanningDataSource, (t) => {+ t.dropUnique(["projectId", "name"]);+ });+ await knex.schema.alterTable(TableName.SecretScanningDataSource, (t) => {+ t.renameColumn("connectionId", "appConnectionId");+ t.timestamp("deletedAt").nullable();+ });+ // Keeps knex's derived name in step with the column, so a later dropForeign(["appConnectionId"]) finds it.+ await renameConstraint(+ knex,+ TableName.SecretScanningDataSource,+ "secret_scanning_data_sources_connectionid_foreign",+ "secret_scanning_data_sources_appconnectionid_foreign"+ );+ await knex.schema.alterTable(TableName.SecretScanningDataSource, (t) => {+ t.index("appConnectionId");+ });+ await knex.raw(`CREATE UNIQUE INDEX ?? ON ?? ("projectId", "name") WHERE "deletedAt" IS NULL`, [+ SOURCE_ACTIVE_NAME_UNIQUE,+ TableName.SecretScanningDataSource+ ]);+ }++ if (await knex.schema.hasColumn(TableName.SecretScanningResource, "dataSourceId")) {+ await knex.schema.alterTable(TableName.SecretScanningResource, (t) => {+ t.dropUnique(["dataSourceId", "externalId"]);+ });+ await knex.schema.alterTable(TableName.SecretScanningResource, (t) => {+ t.renameColumn("dataSourceId", "sourceId");+ t.dropColumn("type");+ t.timestamp("deletedAt").nullable();+ });+ await renameConstraint(+ knex,+ TableName.SecretScanningResource,+ "secret_scanning_resources_datasourceid_foreign",+ "secret_scanning_resources_sourceid_foreign"+ );+ await knex.raw(`CREATE UNIQUE INDEX ?? ON ?? ("sourceId", "externalId") WHERE "deletedAt" IS NULL`, [+ RESOURCE_ACTIVE_EXTERNAL_ID_UNIQUE,+ TableName.SecretScanningResource+ ]);+ }++ if (await knex.schema.hasColumn(TableName.SecretScanningScan, "scanningStartedAt")) {+ await knex.schema.alterTable(TableName.SecretScanningScan, (t) => {+ t.renameColumn("scanningStartedAt", "startedAt");+ });+ await knex.schema.alterTable(TableName.SecretScanningScan, (t) => {+ t.string("trigger").nullable();+ t.uuid("triggeredByUserId").nullable();+ t.foreign("triggeredByUserId").references("id").inTable(TableName.Users).onDelete("SET NULL");+ t.timestamp("completedAt").nullable();+ t.index("resourceId");+ });+ await knex.raw(`ALTER INDEX IF EXISTS ?? RENAME TO ??`, [SCAN_STARTED_AT_LEGACY_INDEX, SCAN_STARTED_AT_INDEX]);+ await knex.raw(`CREATE INDEX ?? ON ?? ("triggeredByUserId") WHERE "triggeredByUserId" IS NOT NULL`, [+ SCAN_TRIGGERED_BY_INDEX,+ TableName.SecretScanningScan+ ]);++ // Every existing full scan was started by someone creating a source or pressing scan, so they count as+ // manual; diff scans only ever came from a push webhook. SET expressions all read the pre-update row.+ await knex.raw(+ `UPDATE ?? SET+ type = CASE type WHEN 'full-scan' THEN 'historical' WHEN 'diff-scan' THEN 'realtime' ELSE type END,+ "trigger" = CASE type WHEN 'diff-scan' THEN 'push' ELSE 'manual' END,+ "createdAt" = COALESCE("createdAt", "startedAt", "progressUpdatedAt", NOW())`,+ [TableName.SecretScanningScan]+ );++ await knex.schema.alterTable(TableName.SecretScanningScan, (t) => {+ t.string("trigger").notNullable().alter();+ t.timestamp("createdAt").notNullable().defaultTo(knex.fn.now()).alter();+ });+ }++ if (await knex.schema.hasColumn(TableName.SecretScanningFinding, "projectId")) {+ await knex.schema.alterTable(TableName.SecretScanningFinding, (t) => {+ t.uuid("resourceId").nullable();+ });++ await knex.raw(`UPDATE ?? AS f SET "resourceId" = s."resourceId" FROM ?? AS s WHERE s.id = f."scanId"`, [+ TableName.SecretScanningFinding,+ TableName.SecretScanningScan+ ]);++ // Every finding is inserted with its scan, and a scan only disappears when its resource is deleted+ // (SET NULL on scanId), so a finding with no resource here belongs to a resource that no longer exists.+ await knex(TableName.SecretScanningFinding).whereNull("resourceId").delete();++ await knex.schema.alterTable(TableName.SecretScanningFinding, (t) => {+ t.dropUnique(["projectId", "fingerprint"]);+ t.dropForeign(["scanId"]);+ });++ await knex.schema.alterTable(TableName.SecretScanningFinding, (t) => {+ t.uuid("resourceId").notNullable().alter();+ t.uuid("scanId").notNullable().alter();+ });++ await knex.schema.alterTable(TableName.SecretScanningFinding, (t) => {+ t.foreign("resourceId").references("id").inTable(TableName.SecretScanningResource).onDelete("CASCADE");+ t.unique(["resourceId", "fingerprint"]);+ t.foreign("scanId").references("id").inTable(TableName.SecretScanningScan).onDelete("CASCADE");+ t.index("scanId");++ t.renameColumn("rule", "ruleKey");+ t.renameColumn("remarks", "triageComment");+ });++ await knex.schema.alterTable(TableName.SecretScanningFinding, (t) => {+ t.text("triageComment").nullable().alter();+ t.string("confidence").nullable();+ t.uuid("triagedByUserId").nullable();+ t.foreign("triagedByUserId").references("id").inTable(TableName.Users).onDelete("SET NULL");+ t.timestamp("triagedAt").nullable();+ t.timestamp("resolvedAt").nullable();+ t.binary("encryptedFinding").nullable();++ t.dropColumn("dataSourceName");+ t.dropColumn("dataSourceType");+ t.dropColumn("resourceName");+ t.dropColumn("resourceType");+ t.dropColumn("projectId");+ });++ await knex.raw(`CREATE INDEX ?? ON ?? ("triagedByUserId") WHERE "triagedByUserId" IS NOT NULL`, [+ FINDING_TRIAGED_BY_INDEX,+ TableName.SecretScanningFinding+ ]);+ }+}++export async function down(knex: Knex): Promise<void> {+ await knex.raw("SET LOCAL lock_timeout = '10s'");++ if (await knex.schema.hasColumn(TableName.SecretScanningFinding, "resourceId")) {+ await knex.schema.alterTable(TableName.SecretScanningFinding, (t) => {+ t.string("projectId").nullable();+ t.string("dataSourceName").nullable();+ t.string("dataSourceType").nullable();+ t.string("resourceName").nullable();+ t.string("resourceType").nullable();+ });++ await knex.raw(+ `UPDATE ?? AS f SET+ "projectId" = src."projectId",+ "dataSourceName" = src.name,+ "dataSourceType" = src.type,+ "resourceName" = r.name,+ "resourceType" = CASE src.type WHEN 'gitlab' THEN 'project' ELSE 'repository' END+ FROM ?? AS r+ JOIN ?? AS src ON src.id = r."sourceId"+ WHERE r.id = f."resourceId"`,+ [TableName.SecretScanningFinding, TableName.SecretScanningResource, TableName.SecretScanningDataSource]+ );++ // The old key is per project, so the same fingerprint in two repos of one project collapses to its oldest row.+ await knex.raw(+ `DELETE FROM ?? AS f USING ?? AS older+ WHERE older."projectId" = f."projectId"+ AND older.fingerprint = f.fingerprint+ AND (older."createdAt", older.id) < (f."createdAt", f.id)`,+ [TableName.SecretScanningFinding, TableName.SecretScanningFinding]+ );++ await knex.raw(`UPDATE ?? SET "triageComment" = LEFT("triageComment", 255) WHERE LENGTH("triageComment") > 255`, [+ TableName.SecretScanningFinding+ ]);++ await knex.raw(`DROP INDEX IF EXISTS ??`, [FINDING_TRIAGED_BY_INDEX]);++ await knex.schema.alterTable(TableName.SecretScanningFinding, (t) => {+ t.dropUnique(["resourceId", "fingerprint"]);+ t.dropForeign(["scanId"]);+ t.dropIndex(["scanId"]);+ t.dropForeign(["resourceId"]);+ t.dropForeign(["triagedByUserId"]);+ });++ await knex.schema.alterTable(TableName.SecretScanningFinding, (t) => {+ t.dropColumn("resourceId");+ t.dropColumn("confidence");+ t.dropColumn("triagedByUserId");+ t.dropColumn("triagedAt");+ t.dropColumn("resolvedAt");+ t.dropColumn("encryptedFinding");++ t.renameColumn("ruleKey", "rule");+ t.renameColumn("triageComment", "remarks");+ });++ await knex.schema.alterTable(TableName.SecretScanningFinding, (t) => {+ t.string("remarks").nullable().alter();+ t.uuid("scanId").nullable().alter();+ t.string("projectId").notNullable().alter();+ t.string("dataSourceName").notNullable().alter();+ t.string("dataSourceType").notNullable().alter();+ t.string("resourceName").notNullable().alter();+ t.string("resourceType").notNullable().alter();+ });++ await knex.schema.alterTable(TableName.SecretScanningFinding, (t) => {+ t.foreign("projectId").references("id").inTable(TableName.Project).onDelete("CASCADE");+ t.foreign("scanId").references("id").inTable(TableName.SecretScanningScan).onDelete("SET NULL");+ t.unique(["projectId", "fingerprint"]);+ });+ }++ if (await knex.schema.hasColumn(TableName.SecretScanningScan, "startedAt")) {+ await knex.raw(+ `UPDATE ?? SET type = CASE type WHEN 'historical' THEN 'full-scan' WHEN 'realtime' THEN 'diff-scan' ELSE type END`,+ [TableName.SecretScanningScan]+ );++ await knex.raw(`DROP INDEX IF EXISTS ??`, [SCAN_TRIGGERED_BY_INDEX]);+ await knex.raw(`ALTER INDEX IF EXISTS ?? RENAME TO ??`, [SCAN_STARTED_AT_INDEX, SCAN_STARTED_AT_LEGACY_INDEX]);++ await knex.schema.alterTable(TableName.SecretScanningScan, (t) => {+ t.dropForeign(["triggeredByUserId"]);+ t.dropIndex(["resourceId"]);+ });+ await knex.schema.alterTable(TableName.SecretScanningScan, (t) => {+ t.dropColumn("trigger");+ t.dropColumn("triggeredByUserId");+ t.dropColumn("completedAt");+ t.renameColumn("startedAt", "scanningStartedAt");+ });+ await knex.schema.alterTable(TableName.SecretScanningScan, (t) => {+ t.timestamp("createdAt").nullable().defaultTo(knex.fn.now()).alter();+ });+ }++ if (await knex.schema.hasColumn(TableName.SecretScanningResource, "sourceId")) {+ // Pre-migration code has no notion of a deleted resource and would treat these as live again.+ await knex(TableName.SecretScanningResource).whereNotNull("deletedAt").delete();++ await knex.raw(`DROP INDEX IF EXISTS ??`, [RESOURCE_ACTIVE_EXTERNAL_ID_UNIQUE]);+ await knex.schema.alterTable(TableName.SecretScanningResource, (t) => {+ t.string("type").nullable();+ });+ await knex.raw(+ `UPDATE ?? AS r SET type = CASE src.type WHEN 'gitlab' THEN 'project' ELSE 'repository' END+ FROM ?? AS src WHERE src.id = r."sourceId"`,+ [TableName.SecretScanningResource, TableName.SecretScanningDataSource]+ );+ await renameConstraint(+ knex,+ TableName.SecretScanningResource,+ "secret_scanning_resources_sourceid_foreign",+ "secret_scanning_resources_datasourceid_foreign"+ );+ await knex.schema.alterTable(TableName.SecretScanningResource, (t) => {+ t.dropColumn("deletedAt");+ t.renameColumn("sourceId", "dataSourceId");+ });+ await knex.schema.alterTable(TableName.SecretScanningResource, (t) => {+ t.string("type").notNullable().alter();+ t.unique(["dataSourceId", "externalId"]);+ });+ }++ if (await knex.schema.hasColumn(TableName.SecretScanningDataSource, "appConnectionId")) {+ await knex(TableName.SecretScanningDataSource).whereNotNull("deletedAt").delete();++ await knex.raw(`DROP INDEX IF EXISTS ??`, [SOURCE_ACTIVE_NAME_UNIQUE]);+ await knex.schema.alterTable(TableName.SecretScanningDataSource, (t) => {+ t.dropIndex(["appConnectionId"]);+ });+ await renameConstraint(+ knex,+ TableName.SecretScanningDataSource,+ "secret_scanning_data_sources_appconnectionid_foreign",+ "secret_scanning_data_sources_connectionid_foreign"+ );+ await knex.schema.alterTable(TableName.SecretScanningDataSource, (t) => {+ t.dropColumn("deletedAt");+ t.renameColumn("appConnectionId", "connectionId");+ });+ await knex.schema.alterTable(TableName.SecretScanningDataSource, (t) => {+ t.unique(["projectId", "name"]);+ });+ }+}diff --git a/backend/src/db/schemas/secret-scanning-data-sources.ts b/backend/src/db/schemas/secret-scanning-data-sources.tsindex d79b45e790..515683f3ae 100644--- a/backend/src/db/schemas/secret-scanning-data-sources.ts+++ b/backend/src/db/schemas/secret-scanning-data-sources.ts@@ -17,12 +17,13 @@ export const SecretScanningDataSourcesSchema = z.object({type: z.string(),config: z.unknown(),encryptedCredentials: zodBuffer.nullable().optional(),- connectionId: z.string().uuid().nullable().optional(),+ appConnectionId: z.string().uuid().nullable().optional(),isAutoScanEnabled: z.boolean().default(true).nullable().optional(),projectId: z.string(),createdAt: z.date(),updatedAt: z.date(),- isDisconnected: z.boolean().default(false)+ isDisconnected: z.boolean().default(false),+ deletedAt: z.date().nullable().optional()});export type TSecretScanningDataSources = z.infer<typeof SecretScanningDataSourcesSchema>;diff --git a/backend/src/db/schemas/secret-scanning-findings.ts b/backend/src/db/schemas/secret-scanning-findings.tsindex c36f229f83..5880e31d61 100644--- a/backend/src/db/schemas/secret-scanning-findings.ts+++ b/backend/src/db/schemas/secret-scanning-findings.ts@@ -5,24 +5,27 @@import { z } from "zod";+import { zodBuffer } from "@app/lib/zod";+import { TImmutableDBKeys } from "./models";export const SecretScanningFindingsSchema = z.object({id: z.string().uuid(),- dataSourceName: z.string(),- dataSourceType: z.string(),- resourceName: z.string(),- resourceType: z.string(),- rule: z.string(),+ ruleKey: z.string(),severity: z.string(),status: z.string().default("unresolved"),- remarks: z.string().nullable().optional(),+ triageComment: z.string().nullable().optional(),fingerprint: z.string(),details: z.unknown(),- projectId: z.string(),- scanId: z.string().uuid().nullable().optional(),+ scanId: z.string().uuid(),createdAt: z.date(),- updatedAt: z.date()+ updatedAt: z.date(),+ resourceId: z.string().uuid(),+ confidence: z.string().nullable().optional(),+ triagedByUserId: z.string().uuid().nullable().optional(),+ triagedAt: z.date().nullable().optional(),+ resolvedAt: z.date().nullable().optional(),+ encryptedFinding: zodBuffer.nullable().optional()});export type TSecretScanningFindings = z.infer<typeof SecretScanningFindingsSchema>;diff --git a/backend/src/db/schemas/secret-scanning-resources.ts b/backend/src/db/schemas/secret-scanning-resources.tsindex e791e1cd62..610bccb07a 100644--- a/backend/src/db/schemas/secret-scanning-resources.ts+++ b/backend/src/db/schemas/secret-scanning-resources.ts@@ -11,10 +11,10 @@ export const SecretScanningResourcesSchema = z.object({id: z.string().uuid(),externalId: z.string(),name: z.string(),- type: z.string(),- dataSourceId: z.string().uuid(),+ sourceId: z.string().uuid(),createdAt: z.date(),- updatedAt: z.date()+ updatedAt: z.date(),+ deletedAt: z.date().nullable().optional()});export type TSecretScanningResources = z.infer<typeof SecretScanningResourcesSchema>;diff --git a/backend/src/db/schemas/secret-scanning-scans.ts b/backend/src/db/schemas/secret-scanning-scans.tsindex 51fdc8dcdc..71f3770730 100644--- a/backend/src/db/schemas/secret-scanning-scans.ts+++ b/backend/src/db/schemas/secret-scanning-scans.ts@@ -13,11 +13,14 @@ export const SecretScanningScansSchema = z.object({statusMessage: z.string().nullable().optional(),type: z.string(),resourceId: z.string().uuid(),- createdAt: z.date().nullable().optional(),- scanningStartedAt: z.date().nullable().optional(),+ createdAt: z.date(),+ startedAt: z.date().nullable().optional(),lastScannedCommit: z.string().nullable().optional(),progressUpdatedAt: z.date().nullable().optional(),- lastScannedCommitDigest: z.string().nullable().optional()+ lastScannedCommitDigest: z.string().nullable().optional(),+ trigger: z.string(),+ triggeredByUserId: z.string().uuid().nullable().optional(),+ completedAt: z.date().nullable().optional()});export type TSecretScanningScans = z.infer<typeof SecretScanningScansSchema>;