Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.
TypeScript acmeagentsai-agentscertificate-managementdevsecopsenvironment-variableskubernetesopen-sourcepampkipostgresprivate-caprivileged-access-managementsecret-managementsecret-managersecretssecrets-managementsecuritysecurity-toolsvault
fix(secret-sync): stop dropping imports shared by two folders in a recursive sync
Calvin2df5ff304c74
diff --git a/backend/src/services/secret-sync/secret-sync-fns.ts b/backend/src/services/secret-sync/secret-sync-fns.tsindex b24448d382..06b139ca8b 100644--- a/backend/src/services/secret-sync/secret-sync-fns.ts+++ b/backend/src/services/secret-sync/secret-sync-fns.ts@@ -84,7 +84,7 @@ import { RailwaySyncFns } from "./railway/railway-sync-fns";import { RENDER_SYNC_LIST_OPTION, RenderSyncFns } from "./render";import { RUNDECK_SYNC_LIST_OPTION, RundeckSyncFns } from "./rundeck";import { SECRET_SYNC_PLAN_MAP } from "./secret-sync-maps";-import { TSecretSyncPayload } from "./secret-sync-payload";+import { createSecretSyncPayload, TSecretSyncPayload } from "./secret-sync-payload";import { SNOWFLAKE_SYNC_LIST_OPTION, SnowflakeSyncFns } from "./snowflake";import { SPACELIFT_SYNC_LIST_OPTION, SpaceliftSyncFns } from "./spacelift";import { SUPABASE_SYNC_LIST_OPTION, SupabaseSyncFns } from "./supabase";@@ -290,8 +290,16 @@ export const SecretSyncFns = {switch (secretSync.destination) {case SecretSync.AWSParameterStore:return AwsParameterStoreSyncFns.syncSecrets(secretSync, payload);- case SecretSync.AWSSecretsManager:- return AwsSecretsManagerSyncFns.syncSecrets(secretSync, payload, payload);+ case SecretSync.AWSSecretsManager: {+ // The many-to-one mapping combines every secret into one AWS secret's JSON body under+ // its own (unprefixed) key, so this destination genuinely needs both views: the+ // schema-applied map for the one-to-one path, and a schema-free one for that JSON body.+ // Re-flatten with no schema to keep this view's duplicate-name detection.+ const unmodifiedPayload = createSecretSyncPayload(payload.all(), {+ environment: secretSync.environment?.slug || ""+ });+ return AwsSecretsManagerSyncFns.syncSecrets(secretSync, payload.flatten(), unmodifiedPayload.flatten());+ }case SecretSync.GitHub:return GithubSyncFns.syncSecrets(secretSync, payload, gatewayService, gatewayV2Service, gatewayPoolService, {gitHubAppDAL,@@ -387,10 +395,16 @@ export const SecretSyncFns = {return CircleCISyncFns.syncSecrets(secretSync, payload);case SecretSync.AzureEntraIdScim:return AzureEntraIdScimSyncFns.syncSecrets(secretSync, payload, { appConnectionDAL, kmsService });- case SecretSync.ExternalInfisical:- // Key schema is intentionally not applied for Infisical-to-Infisical syncs to prevent- // infinite sync loops where the prefixed key triggers another sync cycle.- return ExternalInfisicalSyncFns.syncSecrets(secretSync, payload);+ case SecretSync.ExternalInfisical: {+ // Key schema must never be applied for Infisical-to-Infisical syncs, or the prefixed+ // key triggers another sync cycle. Re-flatten with no schema rather than passing the+ // payload through, since payload.flatten() would apply the schema this destination+ // carries for every other provider.+ const unschematizedPayload = createSecretSyncPayload(payload.all(), {+ environment: secretSync.environment?.slug || ""+ });+ return ExternalInfisicalSyncFns.syncSecrets(secretSync, unschematizedPayload.flatten());+ }case SecretSync.OVH:return OvhSyncFns.syncSecrets(secretSync, payload);case SecretSync.Devin:@@ -710,10 +724,14 @@ export const SecretSyncFns = {return CircleCISyncFns.removeSecrets(secretSync, payload);case SecretSync.AzureEntraIdScim:return AzureEntraIdScimSyncFns.removeSecrets();- case SecretSync.ExternalInfisical:- // Key schema is intentionally not applied for Infisical-to-Infisical syncs to prevent- // infinite sync loops where the prefixed key triggers another sync cycle.- return ExternalInfisicalSyncFns.removeSecrets(secretSync, payload);+ case SecretSync.ExternalInfisical: {+ // See the matching case in syncSecrets: this destination always gets the raw,+ // unschematized view, never payload.flatten().+ const unschematizedPayload = createSecretSyncPayload(payload.all(), {+ environment: secretSync.environment?.slug || ""+ });+ return ExternalInfisicalSyncFns.removeSecrets(secretSync, unschematizedPayload.flatten());+ }case SecretSync.OVH:return OvhSyncFns.removeSecrets(secretSync, payload);case SecretSync.Devin:diff --git a/backend/src/services/secret-sync/secret-sync-queue.ts b/backend/src/services/secret-sync/secret-sync-queue.tsindex 30d3f0a9d0..203da200d7 100644--- a/backend/src/services/secret-sync/secret-sync-queue.ts+++ b/backend/src/services/secret-sync/secret-sync-queue.ts@@ -435,8 +435,8 @@ export const secretSyncQueueFactory = ({const importedSecretMap: TSecretMap = {};- // Task 5 blocks combining import behavior with a recursive sync, so this payload always- // covers exactly one folder. Comparison is against raw keys as returned by the destination+ // Import behavior is never combined with a recursive sync, so this payload always covers+ // exactly one folder. Comparison is against raw keys as returned by the destination// (already schema-stripped), so entries come from all() rather than the schema-applying flatten().const payload = await $getInfisicalSecrets(secretSync, false);const secretMap: TSecretMap = Object.fromEntries(diff --git a/backend/src/services/secret-sync/secret-sync-recursive-fns.test.ts b/backend/src/services/secret-sync/secret-sync-recursive-fns.test.tsindex fbfd9dc455..3fb0df1d06 100644--- a/backend/src/services/secret-sync/secret-sync-recursive-fns.test.ts+++ b/backend/src/services/secret-sync/secret-sync-recursive-fns.test.ts@@ -1,4 +1,10 @@-import { assertWithinSecretLimit, mergeImportedSecrets, resolveSyncFolders, SECRET_SYNC_MAX_SECRETS } from "./secret-sync-recursive-fns";+import {+ assertWithinSecretLimit,+ buildSyncPayload,+ mergeImportedSecrets,+ resolveSyncFolders,+ SECRET_SYNC_MAX_SECRETS+} from "./secret-sync-recursive-fns";const deps = {folderDAL: {@@ -88,3 +94,96 @@ describe("mergeImportedSecrets", () => {expect(merged[0].value).toBe("second");});});++describe("buildSyncPayload", () => {+ test("two folders importing the same source both receive its secrets", async () => {+ const sharedSecret = {+ id: "secret-1",+ key: "DB_URL",+ folderId: "shared-folder",+ encryptedValue: Buffer.from("shared-value"),+ encryptedComment: null,+ skipMultilineEncoding: false,+ tags: [],+ secretMetadata: []+ };++ const rootImport = {+ id: "import-root",+ folderId: "root",+ importPath: "/shared",+ importEnv: { id: "env-1", slug: "dev", name: "dev", projectId: "proj-1" },+ isReplication: false,+ isReserved: false,+ position: 1+ };++ const apiImport = {+ id: "import-api",+ folderId: "api",+ importPath: "/shared",+ importEnv: { id: "env-1", slug: "dev", name: "dev", projectId: "proj-1" },+ isReplication: false,+ isReserved: false,+ position: 1+ };++ const buildDeps = {+ folderDAL: {+ find: async () => [+ { id: "root", name: "root", parentId: null, envId: "env-1", isReserved: false },+ { id: "api", name: "api", parentId: "root", envId: "env-1", isReserved: false }+ ],+ findByManySecretPath: async (query: { envId: string; secretPath: string }[]) =>+ query.map(() => ({+ id: "shared-folder",+ envId: "env-1",+ path: "/shared",+ name: "shared",+ parentId: null,+ isReserved: false+ }))+ },+ projectEnvDAL: { findOne: async () => ({ id: "env-1", slug: "dev", projectId: "proj-1" }) },+ secretV2BridgeDAL: {+ findByFolderIds: async () => [],+ find: async () => [sharedSecret]+ },+ secretImportDAL: {+ findByFolderIds: async (folderIds: string[]) =>+ [rootImport, apiImport].filter((row) => folderIds.includes(row.folderId)),+ findByIds: async () => []+ },+ expandSecretReferences: async ({ value }: { value?: string }) => value,+ decryptSecretValue: (value?: Buffer | null) => (value ? value.toString() : ""),+ fnSecretsV2FromImportsDeps: {+ projectFolderGrantDAL: { find: async () => [] },+ actorOrgId: "org-1",+ orgDAL: { findOrgById: async () => ({ allowCrossProjectSecretSharing: false }) },+ licenseService: { getPlan: async () => ({ crossProjectSecretSharing: false }) },+ kmsService: {+ createCipherPairWithDataKey: async () => ({+ decryptor: () => "",+ encryptor: () => ({ cipherTextBlob: Buffer.from("") })+ })+ }+ }+ } as unknown as Parameters<typeof buildSyncPayload>[0];++ const payload = await buildSyncPayload(buildDeps, {+ projectId: "proj-1",+ environment: "dev",+ sourcePath: "/",+ sourceFolderId: "root",+ recursive: true,+ includeImports: true+ });++ const paths = payload+ .all()+ .map((entry) => entry.path)+ .sort();++ expect(paths).toEqual(["/", "/api"]);+ });+});diff --git a/backend/src/services/secret-sync/secret-sync-recursive-fns.ts b/backend/src/services/secret-sync/secret-sync-recursive-fns.tsindex 8f78854838..c3789b57a8 100644--- a/backend/src/services/secret-sync/secret-sync-recursive-fns.ts+++ b/backend/src/services/secret-sync/secret-sync-recursive-fns.ts@@ -1,4 +1,5 @@import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";+import { groupBy } from "@app/lib/fn";import { TKmsServiceFactory } from "@app/services/kms/kms-service";import { TOrgDALFactory } from "@app/services/org/org-dal";import { TProjectEnvDALFactory } from "@app/services/project-env/project-env-dal";@@ -108,7 +109,7 @@ export const buildSyncPayload = async (deps: {folderDAL: Pick<TSecretFolderDALFactory, "find" | "findByManySecretPath">;projectEnvDAL: Pick<TProjectEnvDALFactory, "findOne">;- secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "findByFolderIds" | "find">;+ secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "findByFolderId" | "findByFolderIds" | "find">;secretImportDAL: Pick<TSecretImportDALFactory, "findByFolderIds" | "findByIds">;expandSecretReferences: TExpandSecretReferences;decryptSecretValue: (value?: Buffer | null) => string;@@ -140,7 +141,14 @@ export const buildSyncPayload = async (const pathByFolderId = new Map(folders.map(({ folderId, path }) => [folderId, path]));- const secrets = await secretV2BridgeDAL.findByFolderIds({ folderIds: folders.map(({ folderId }) => folderId) });+ // findByFolderIds joins tags, metadata, rotation, honey token, reminder, recipients and users+ // plus a DENSE_RANK window, so it costs far more than the three narrow queries findByFolderId+ // runs. Every non-recursive sync (the vast majority today) hits this path, so it keeps using+ // the cheaper single-folder read; only a genuinely multi-folder subtree pays for the join.+ const secrets =+ folders.length === 1+ ? await secretV2BridgeDAL.findByFolderId({ folderId: folders[0].folderId })+ : await secretV2BridgeDAL.findByFolderIds({ folderIds: folders.map(({ folderId }) => folderId) });assertWithinSecretLimit(secrets.length);@@ -182,18 +190,31 @@ export const buildSyncPayload = async (let allEntries = entries;if (secretImports.length) {- const importedSecrets = await fnSecretsV2FromImports({- decryptor: decryptSecretValue,- folderDAL,- secretDAL: secretV2BridgeDAL,- expandSecretReferences,- secretImportDAL,- secretImports,- hasSecretAccess: () => true,- viewSecretValue: true,- projectId,- ...deps.fnSecretsV2FromImportsDeps- });+ // fnSecretsV2FromImports dedupes by (importEnv, importPath) across the whole batch it is+ // given, keeping one result row per source. Two folders importing the same source in one+ // batched call would collapse to a single importFolderId, silently dropping the other+ // folder's copy from the payload. Calling it once per declaring folder keeps each call's+ // dedup scoped to that folder's own imports, so every declaring folder gets its own result.+ const importsByDeclaringFolder = groupBy(secretImports, (secretImport) => secretImport.folderId);++ const importedSecrets = (+ await Promise.all(+ Object.values(importsByDeclaringFolder).map((declaringFolderImports) =>+ fnSecretsV2FromImports({+ decryptor: decryptSecretValue,+ folderDAL,+ secretDAL: secretV2BridgeDAL,+ expandSecretReferences,+ secretImportDAL,+ secretImports: declaringFolderImports,+ hasSecretAccess: () => true,+ viewSecretValue: true,+ projectId,+ ...deps.fnSecretsV2FromImportsDeps+ })+ )+ )+ ).flat();allEntries = mergeImportedSecrets(entries,