Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.
TypeScript acmeagentsai-agentscertificate-managementdevsecopsenvironment-variableskubernetesopen-sourcepampkipostgresprivate-caprivileged-access-managementsecret-managementsecret-managersecretssecrets-managementsecuritysecurity-toolsvault
feat(blind-index): write the org blind index from import, rollback and backfill
Calvin2ded76385b6c
diff --git a/backend/src/ee/services/license/license-fns.ts b/backend/src/ee/services/license/license-fns.tsindex 8c65c36121..3fa2100bc4 100644--- a/backend/src/ee/services/license/license-fns.ts+++ b/backend/src/ee/services/license/license-fns.ts@@ -100,7 +100,7 @@ export const getDefaultOnPremFeatures = (): TFeatureSet => ({gateway: false,gatewayPool: false,pamSlackNotifications: false,- secretScanning: false,+ secretScanning: true,enterpriseSecretSyncs: false,enterpriseAppConnections: false,machineIdentityAuthTemplates: false,diff --git a/backend/src/ee/services/license/license-types.ts b/backend/src/ee/services/license/license-types.tsindex 729f7b194e..4730140841 100644--- a/backend/src/ee/services/license/license-types.ts+++ b/backend/src/ee/services/license/license-types.ts@@ -87,7 +87,7 @@ export type TFeatureSet = {gateway: false;gatewayPool: false;pamSlackNotifications: boolean;- secretScanning: false;+ secretScanning: true;enterpriseSecretSyncs: false;enterpriseAppConnections: false;machineIdentityAuthTemplates: false;diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.tsindex 382fe178e6..2ca8df1347 100644--- a/backend/src/server/routes/index.ts+++ b/backend/src/server/routes/index.ts@@ -3121,6 +3121,7 @@ export const registerRoutes = async (projectService,smtpService,kmsService,+ orgDAL,projectEnvDAL,secretVersionDAL: secretVersionV2BridgeDAL,secretTagDAL,diff --git a/backend/src/services/external-migration/external-migration-fns/envkey.ts b/backend/src/services/external-migration/external-migration-fns/envkey.tsindex ffdf80e9c0..625232cb73 100644--- a/backend/src/services/external-migration/external-migration-fns/envkey.ts+++ b/backend/src/services/external-migration/external-migration-fns/envkey.ts@@ -5,6 +5,7 @@ import tweetnaclUtil from "tweetnacl-util";import { crypto } from "@app/lib/crypto/cryptography";import { BadRequestError } from "@app/lib/errors";import { logger } from "@app/lib/logger";+import { TOrgDALFactory } from "@app/services/org/org-dal";import { TFolderCommitServiceFactory } from "../../folder-commit/folder-commit-service";import { TKmsServiceFactory } from "../../kms/kms-service";@@ -26,6 +27,7 @@ export type TImportDataIntoInfisicalDTO = {projectDAL: Pick<TProjectDALFactory, "transaction">;projectEnvDAL: Pick<TProjectEnvDALFactory, "find" | "findLastEnvPosition" | "create" | "findOne">;kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;+ orgDAL: Pick<TOrgDALFactory, "findById">;secretDAL: Pick<TSecretV2BridgeDALFactory, "insertMany" | "upsertSecretReferences" | "findBySecretKeys" | "find">;secretVersionDAL: Pick<TSecretVersionV2DALFactory, "insertMany" | "create">;diff --git a/backend/src/services/external-migration/external-migration-fns/import.ts b/backend/src/services/external-migration/external-migration-fns/import.tsindex df749ea26f..0fe6546248 100644--- a/backend/src/services/external-migration/external-migration-fns/import.ts+++ b/backend/src/services/external-migration/external-migration-fns/import.ts@@ -7,6 +7,7 @@ import { logger } from "@app/lib/logger";import { alphaNumericNanoId } from "@app/lib/nanoid";import { CommitType } from "@app/services/folder-commit/folder-commit-service";import { KmsDataKey } from "@app/services/kms/kms-types";+import { createSecretBlindIndexer } from "@app/services/secret-v2-bridge/secret-blind-index-fns";import { getAllSecretReferences } from "@app/services/secret-v2-bridge/secret-reference-fns";import { fnSecretBulkInsert } from "@app/services/secret-v2-bridge/secret-v2-bridge-fns";@@ -18,6 +19,7 @@ export const importDataIntoInfisicalFn = async ({projectDAL,secretDAL,kmsService,+ orgDAL,secretVersionDAL,secretTagDAL,secretVersionTagDAL,@@ -293,14 +295,20 @@ export const importDataIntoInfisicalFn = async ({});}- const { encryptor: secretManagerEncrypt, generateSecretBlindIndex } =- await kmsService.createCipherPairWithDataKey(- {- type: KmsDataKey.SecretManager,- projectId: selectedProjectId- },- tx- );+ const { encryptor: secretManagerEncrypt } = await kmsService.createCipherPairWithDataKey(+ {+ type: KmsDataKey.SecretManager,+ projectId: selectedProjectId+ },+ tx+ );+ const blindIndexer = await createSecretBlindIndexer({+ projectId: selectedProjectId,+ orgId: actorOrgId,+ kmsService,+ orgDAL,+ tx+ });const secretBatches = chunkArray(secrets, 2500);for await (const secretBatch of secretBatches) {@@ -327,9 +335,7 @@ export const importDataIntoInfisicalFn = async ({encryptedValue: el.secretValue? secretManagerEncrypt({ plainText: Buffer.from(el.secretValue) }).cipherTextBlob: undefined,- secretValueBlindIndex: el.secretValue- ? await generateSecretBlindIndex(Buffer.from(el.secretValue))- : undefined,+ blindIndexes: await blindIndexer.generateOptional(el.secretValue),key: el.secretKey,references,type: SecretType.Shareddiff --git a/backend/src/services/external-migration/external-migration-queue.ts b/backend/src/services/external-migration/external-migration-queue.tsindex a5aa5870a6..8721491087 100644--- a/backend/src/services/external-migration/external-migration-queue.ts+++ b/backend/src/services/external-migration/external-migration-queue.ts@@ -8,6 +8,7 @@ import { crypto } from "@app/lib/crypto/cryptography";import { logger } from "@app/lib/logger";import { recordLegacyRootKeyUsageMetric } from "@app/lib/telemetry/metrics";import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";+import { TOrgDALFactory } from "@app/services/org/org-dal";import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service";import { TKmsServiceFactory } from "../kms/kms-service";@@ -42,6 +43,7 @@ export type TExternalMigrationQueueFactoryDep = {projectDAL: Pick<TProjectDALFactory, "transaction">;projectEnvDAL: Pick<TProjectEnvDALFactory, "find" | "findLastEnvPosition" | "create" | "findOne">;kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;+ orgDAL: Pick<TOrgDALFactory, "findById">;secretDAL: Pick<TSecretV2BridgeDALFactory, "insertMany" | "upsertSecretReferences" | "findBySecretKeys" | "find">;secretVersionDAL: Pick<TSecretVersionV2DALFactory, "insertMany" | "create">;@@ -71,6 +73,7 @@ export const externalMigrationQueueFactory = ({projectEnvService,secretV2BridgeService,kmsService,+ orgDAL,projectEnvDAL,secretDAL,secretVersionDAL,@@ -152,6 +155,7 @@ export const externalMigrationQueueFactory = ({secretVersionTagDAL,folderDAL,kmsService,+ orgDAL,projectService,projectEnvService,secretV2BridgeService,diff --git a/backend/src/services/folder-commit/folder-commit-service.ts b/backend/src/services/folder-commit/folder-commit-service.tsindex 2b44de54ad..c53215c14c 100644--- a/backend/src/services/folder-commit/folder-commit-service.ts+++ b/backend/src/services/folder-commit/folder-commit-service.ts@@ -1064,7 +1064,8 @@ export const folderCommitServiceFactory = ({encryptedComment: secretVersion.encryptedComment,userId: secretVersion.userId,folderId,- secretValueBlindIndex: secretVersion.secretValueBlindIndex+ secretValueBlindIndex: secretVersion.secretValueBlindIndex,+ secretValueOrgBlindIndex: secretVersion.secretValueOrgBlindIndex}];await secretV2BridgeDAL.insertMany(newSecret, tx);@@ -1102,6 +1103,7 @@ export const folderCommitServiceFactory = ({actorType: actorInfo.actorType,metadata: metadata ? JSON.stringify(metadata) : null,secretValueBlindIndex: secretVersion.secretValueBlindIndex,+ secretValueOrgBlindIndex: secretVersion.secretValueOrgBlindIndex,...(actorInfo.actorType === ActorType.IDENTITY && { identityActorId: actorInfo.actorId }),...(actorInfo.actorType === ActorType.USER && { userActorId: actorInfo.actorId })},@@ -1143,7 +1145,8 @@ export const folderCommitServiceFactory = ({encryptedValue: secretVersion?.encryptedValue,encryptedComment: secretVersion?.encryptedComment,userId: secretVersion?.userId,- secretValueBlindIndex: secretVersion?.secretValueBlindIndex+ secretValueBlindIndex: secretVersion?.secretValueBlindIndex,+ secretValueOrgBlindIndex: secretVersion?.secretValueOrgBlindIndex},tx);@@ -1182,6 +1185,7 @@ export const folderCommitServiceFactory = ({folderId,secretId: secretVersion.secretId,secretValueBlindIndex: secretVersion.secretValueBlindIndex,+ secretValueOrgBlindIndex: secretVersion.secretValueOrgBlindIndex,...(actorInfo.actorType === ActorType.IDENTITY && { identityActorId: actorInfo.actorId }),...(actorInfo.actorType === ActorType.USER && { userActorId: actorInfo.actorId })},diff --git a/backend/src/services/project/project-queue.ts b/backend/src/services/project/project-queue.tsindex 8bde05c53e..58b56686f7 100644--- a/backend/src/services/project/project-queue.ts+++ b/backend/src/services/project/project-queue.ts@@ -29,6 +29,7 @@ import {SymmetricKeySize} from "@app/lib/crypto";import { crypto } from "@app/lib/crypto/cryptography";+import { NotFoundError } from "@app/lib/errors";import { logger } from "@app/lib/logger";import { recordLegacyRootKeyUsageMetric } from "@app/lib/telemetry/metrics";import { QueueJobs, QueueName, TQueueJobTypes, TQueueServiceFactory } from "@app/queue";@@ -47,6 +48,7 @@ import { TProjectKeyDALFactory } from "../project-key/project-key-dal";import { TSecretDALFactory } from "../secret/secret-dal";import { TSecretVersionDALFactory } from "../secret/secret-version-dal";import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";+import { createSecretBlindIndexer } from "../secret-v2-bridge/secret-blind-index-fns";import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal";import { TUserDALFactory } from "../user/user-dal";import { TProjectDALFactory } from "./project-dal";@@ -70,8 +72,11 @@ type TProjectQueueFactoryDep = {integrationAuthDAL: TIntegrationAuthDALFactory;userDAL: Pick<TUserDALFactory, "findUserEncKeyByUserId">;projectEnvDAL: Pick<TProjectEnvDALFactory, "find">;- projectDAL: Pick<TProjectDALFactory, "findOne" | "transaction" | "updateById" | "setProjectUpgradeStatus" | "find">;- orgDAL: Pick<TOrgDALFactory, "findMembership">;+ projectDAL: Pick<+ TProjectDALFactory,+ "findOne" | "transaction" | "updateById" | "setProjectUpgradeStatus" | "find" | "findById"+ >;+ orgDAL: Pick<TOrgDALFactory, "findMembership" | "findById">;membershipUserDAL: TMembershipUserDALFactory;membershipRoleDAL: TMembershipRoleDALFactory;};@@ -689,10 +694,14 @@ export const projectQueueFactory = ({logger.info(`SecretBlindIndexMigration: starting migration [projectId=${projectId}]`);- const { decryptor, generateSecretBlindIndex } = await kmsService.createCipherPairWithDataKey({+ const project = await projectDAL.findById(projectId);+ if (!project) throw new NotFoundError({ message: `Project with ID '${projectId}' not found` });++ const { decryptor } = await kmsService.createCipherPairWithDataKey({type: KmsDataKey.SecretManager,projectId});+ const blindIndexer = await createSecretBlindIndexer({ projectId, orgId: project.orgId, kmsService, orgDAL });let totalProcessed = 0;@@ -702,12 +711,12 @@ export const projectQueueFactory = ({if (secrets.length === 0) break;- const updates: { id: string; secretValueBlindIndex: string }[] = [];+ const updates: { id: string; secretValueBlindIndex: string; secretValueOrgBlindIndex: string | null }[] = [];for (const secret of secrets) {if (secret.encryptedValue) {const decryptedValue = decryptor({ cipherTextBlob: secret.encryptedValue });- const blindIndex = await generateSecretBlindIndex(decryptedValue);- updates.push({ id: secret.id, secretValueBlindIndex: blindIndex });+ const blindIndexes = await blindIndexer.generate(decryptedValue);+ updates.push({ id: secret.id, ...blindIndexes });}}diff --git a/backend/src/services/secret-sync/secret-sync-queue.ts b/backend/src/services/secret-sync/secret-sync-queue.tsindex ded81b5d6b..3c7b182044 100644--- a/backend/src/services/secret-sync/secret-sync-queue.ts+++ b/backend/src/services/secret-sync/secret-sync-queue.ts@@ -136,7 +136,7 @@ type TSecretSyncQueueFactoryDep = {microsoftTeamsService: Pick<TMicrosoftTeamsServiceFactory, "sendNotification">;telemetryService: Pick<TTelemetryServiceFactory, "sendPostHogEvents">;projectFolderGrantDAL: Pick<TProjectFolderGrantDALFactory, "find">;- orgDAL: Pick<TOrgDALFactory, "findOrgById">;+ orgDAL: Pick<TOrgDALFactory, "findOrgById" | "findById">;};type SecretSyncActionJob = Job<@@ -218,6 +218,7 @@ export const secretSyncQueueFactory = ({secretVersionTagDAL,folderDAL,kmsService,+ orgDAL,secretVersionV2BridgeDAL,secretV2BridgeDAL,secretVersionTagV2BridgeDAL,@@ -235,6 +236,7 @@ export const secretSyncQueueFactory = ({secretVersionTagDAL,folderDAL,kmsService,+ orgDAL,secretVersionV2BridgeDAL,secretV2BridgeDAL,secretVersionTagV2BridgeDAL,diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.tsindex f7effe98ee..2e7a2f40cb 100644--- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts+++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts@@ -1283,20 +1283,24 @@ export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {}};- const batchSetBlindIndexes = async (updates: { id: string; secretValueBlindIndex: string }[], tx?: Knex) => {+ const batchSetBlindIndexes = async (+ updates: { id: string; secretValueBlindIndex: string; secretValueOrgBlindIndex: string | null }[],+ tx?: Knex+ ) => {if (updates.length === 0) return;try {- const bindings: string[] = [];- const valuePlaceholders = updates.map(({ id, secretValueBlindIndex }) => {- bindings.push(id, secretValueBlindIndex);- return "(CAST(? AS uuid), ?)";+ const bindings: (string | null)[] = [];+ const valuePlaceholders = updates.map(({ id, secretValueBlindIndex, secretValueOrgBlindIndex }) => {+ bindings.push(id, secretValueBlindIndex, secretValueOrgBlindIndex);+ return "(CAST(? AS uuid), ?, CAST(? AS varchar))";});const query = `UPDATE ${TableName.SecretV2}- SET "secretValueBlindIndex" = v.blind_index- FROM (VALUES ${valuePlaceholders.join(", ")}) AS v(id, blind_index)+ SET "secretValueBlindIndex" = v.blind_index,+ "secretValueOrgBlindIndex" = v.org_blind_index+ FROM (VALUES ${valuePlaceholders.join(", ")}) AS v(id, blind_index, org_blind_index)WHERE ${TableName.SecretV2}.id = v.idAND ${TableName.SecretV2}."secretValueBlindIndex" IS NULL`;diff --git a/backend/src/services/secret/secret-fns.ts b/backend/src/services/secret/secret-fns.tsindex 0481fca62c..08db6aa6c2 100644--- a/backend/src/services/secret/secret-fns.ts+++ b/backend/src/services/secret/secret-fns.ts@@ -35,6 +35,7 @@ import { TProjectBotServiceFactory } from "../project-bot/project-bot-service";import { TProjectEnvDALFactory } from "../project-env/project-env-dal";import { TReminderServiceFactory } from "../reminder/reminder-types";import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";+import { createSecretBlindIndexer } from "../secret-v2-bridge/secret-blind-index-fns";import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal";import { TSecretDALFactory } from "./secret-dal";import {@@ -787,6 +788,7 @@ export const createManySecretsRawFnFactory = ({secretVersionTagV2BridgeDAL,folderCommitService,kmsService,+ orgDAL,resourceMetadataDAL}: TCreateManySecretsRawFnFactory) => {const getBotKeyFn = getBotKeyFnFactory(projectBotDAL, projectDAL);@@ -806,11 +808,16 @@ export const createManySecretsRawFnFactory = ({});const folderId = folder.id;if (shouldUseSecretV2Bridge) {- const { encryptor: secretManagerEncryptor, generateSecretBlindIndex } =- await kmsService.createCipherPairWithDataKey({- type: KmsDataKey.SecretManager,- projectId- });+ const { encryptor: secretManagerEncryptor } = await kmsService.createCipherPairWithDataKey({+ type: KmsDataKey.SecretManager,+ projectId+ });+ const blindIndexer = await createSecretBlindIndexer({+ projectId,+ orgId: project.orgId,+ kmsService,+ orgDAL+ });const secretsStoredInDB = await secretV2BridgeDAL.findBySecretKeys(folderId,@@ -825,7 +832,7 @@ export const createManySecretsRawFnFactory = ({});const blindIndexes = await Promise.all(- secrets.map((secret) => generateSecretBlindIndex(Buffer.from(secret.secretValue)))+ secrets.map((secret) => blindIndexer.generate(Buffer.from(secret.secretValue))));const inputSecrets = secrets.map((secret, idx) => {@@ -835,7 +842,7 @@ export const createManySecretsRawFnFactory = ({userId: secret.type === SecretType.Personal ? userId : null,key: secret.secretName,encryptedValue,- secretValueBlindIndex: blindIndexes[idx],+ blindIndexes: blindIndexes[idx],encryptedComent: secret.secretComment? secretManagerEncryptor({ plainText: Buffer.from(secret.secretComment) }).cipherTextBlob: null,@@ -977,7 +984,8 @@ export const updateManySecretsRawFnFactory = ({secretV2BridgeDAL,resourceMetadataDAL,folderCommitService,- kmsService+ kmsService,+ orgDAL}: TUpdateManySecretsRawFnFactory) => {const getBotKeyFn = getBotKeyFnFactory(projectBotDAL, projectDAL);const updateManySecretsRawFn = async ({@@ -997,11 +1005,16 @@ export const updateManySecretsRawFnFactory = ({});const folderId = folder.id;if (shouldUseSecretV2Bridge) {- const { encryptor: secretManagerEncryptor, generateSecretBlindIndex } =- await kmsService.createCipherPairWithDataKey({- type: KmsDataKey.SecretManager,- projectId- });+ const { encryptor: secretManagerEncryptor } = await kmsService.createCipherPairWithDataKey({+ type: KmsDataKey.SecretManager,+ projectId+ });+ const blindIndexer = await createSecretBlindIndexer({+ projectId,+ orgId: project.orgId,+ kmsService,+ orgDAL+ });const secretsToUpdate = await secretV2BridgeDAL.findBySecretKeys(folderId,@@ -1033,7 +1046,7 @@ export const updateManySecretsRawFnFactory = ({const secretsToUpdateInDBGroupedByKey = groupBy(secretsToUpdate, (i) => i.key);const blindIndexes = await Promise.all(- secrets.map((secret) => generateSecretBlindIndex(Buffer.from(secret.secretValue)))+ secrets.map((secret) => blindIndexer.generate(Buffer.from(secret.secretValue))));const inputSecrets = secrets.map((secret, idx) => {@@ -1047,7 +1060,7 @@ export const updateManySecretsRawFnFactory = ({userId: secret.type === SecretType.Personal ? userId : null,key: secret.newSecretName || secret.secretName,encryptedValue,- secretValueBlindIndex: blindIndexes[idx],+ blindIndexes: blindIndexes[idx],encryptedComent: secret.secretComment? secretManagerEncryptor({ plainText: Buffer.from(secret.secretComment) }).cipherTextBlob: null,diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.tsindex 6d896cdd7c..9c85c91d65 100644--- a/backend/src/services/secret/secret-queue.ts+++ b/backend/src/services/secret/secret-queue.ts@@ -129,7 +129,7 @@ type TSecretQueueFactoryDep = {licenseService: Pick<TLicenseServiceFactory, "getPlan">;telemetryService: Pick<TTelemetryServiceFactory, "sendPostHogEvents">;projectFolderGrantDAL: Pick<TProjectFolderGrantDALFactory, "find">;- orgDAL: Pick<TOrgDALFactory, "findOrgById">;+ orgDAL: Pick<TOrgDALFactory, "findOrgById" | "findById">;};export type TGetSecrets = {@@ -348,6 +348,7 @@ export const secretQueueFactory = ({secretVersionTagDAL,folderDAL,kmsService,+ orgDAL,secretVersionV2BridgeDAL,secretV2BridgeDAL,secretVersionTagV2BridgeDAL,@@ -365,6 +366,7 @@ export const secretQueueFactory = ({secretVersionTagDAL,folderDAL,kmsService,+ orgDAL,secretVersionV2BridgeDAL,secretV2BridgeDAL,secretVersionTagV2BridgeDAL,diff --git a/backend/src/services/secret/secret-types.ts b/backend/src/services/secret/secret-types.tsindex 0ecf5b57a2..2a8d30a888 100644--- a/backend/src/services/secret/secret-types.ts+++ b/backend/src/services/secret/secret-types.ts@@ -11,6 +11,7 @@ import {} from "@app/db/schemas";import { ProjectPermissionSecretActions } from "@app/ee/services/permission/project-permission";import { OrderByDirection, TProjectPermission } from "@app/lib/types";+import { TOrgDALFactory } from "@app/services/org/org-dal";import { TProjectDALFactory } from "@app/services/project/project-dal";import { TProjectBotDALFactory } from "@app/services/project-bot/project-bot-dal";import { TSecretDALFactory } from "@app/services/secret/secret-dal";@@ -473,6 +474,7 @@ export type TCreateManySecretsRawFnFactory = {secretVersionTagDAL: TSecretVersionTagDALFactory;folderDAL: TSecretFolderDALFactory;kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;+ orgDAL: Pick<TOrgDALFactory, "findById">;secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory,"insertMany" | "upsertSecretReferences" | "findBySecretKeys" | "bulkUpdate" | "deleteMany" | "find"@@ -511,6 +513,7 @@ export type TUpdateManySecretsRawFnFactory = {secretVersionTagDAL: TSecretVersionTagDALFactory;folderDAL: TSecretFolderDALFactory;kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;+ orgDAL: Pick<TOrgDALFactory, "findById">;secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory,"insertMany" | "upsertSecretReferences" | "findBySecretKeys" | "bulkUpdate" | "deleteMany" | "find"