Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.
TypeScript acmeagentsai-agentscertificate-managementdevsecopsenvironment-variableskubernetesopen-sourcepampkipostgresprivate-caprivileged-access-managementsecret-managementsecret-managersecretssecrets-managementsecuritysecurity-toolsvault
fix(secret-sync): address review comments on buildSyncPayload and schema docs
Calvin703c733a3406
diff --git a/backend/src/services/secret-sync/secret-sync-queue.ts b/backend/src/services/secret-sync/secret-sync-queue.tsindex ee233665e3..c5b6eca4c5 100644--- a/backend/src/services/secret-sync/secret-sync-queue.ts+++ b/backend/src/services/secret-sync/secret-sync-queue.ts@@ -44,6 +44,7 @@ import {import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";import { enterpriseSyncCheck, parseSyncErrorMessage, SecretSyncFns } from "@app/services/secret-sync/secret-sync-fns";import { SECRET_SYNC_DAILY_RETRY_DESTINATIONS, SECRET_SYNC_NAME_MAP } from "@app/services/secret-sync/secret-sync-maps";+import { createSecretSyncPayload, dedupeEntriesByDestinationKey } from "@app/services/secret-sync/secret-sync-payload";import { buildSyncPayload, getAncestorPaths } from "@app/services/secret-sync/secret-sync-recursive-fns";import {SecretSyncAction,@@ -302,7 +303,7 @@ export const secretSyncQueueFactory = ({kmsService});- return buildSyncPayload(+ const entries = await buildSyncPayload({folderDAL,projectEnvDAL,@@ -323,11 +324,19 @@ export const secretSyncQueueFactory = ({sourcePath: folder.path,sourceFolderId: folderId,recursive: Boolean(syncOptions?.recursive),- keySchema: syncOptions?.keySchema,- includeImports,- dedupeForRemoval+ includeImports});++ // Duplicates across folders are deduped before the payload is built, so flatten() cannot+ // throw here: a sync that has drifted into a duplicate-name state must still be removable.+ // Every other caller keeps the duplicate-name check intact.+ return createSecretSyncPayload(+ dedupeForRemoval+ ? dedupeEntriesByDestinationKey(entries, { environment: environment.slug, keySchema: syncOptions?.keySchema })+ : entries,+ { environment: environment.slug, keySchema: syncOptions?.keySchema }+ );};const queueSecretSyncSyncSecretsById = async (payload: TQueueSecretSyncSyncSecretsByIdDTO) => {@@ -831,8 +840,8 @@ export const secretSyncQueueFactory = ({projectId});- // Duplicates across folders are deduped before the payload is built, so flatten() cannot- // throw here: a sync that has drifted into a duplicate-name state must still be removable.+ // dedupeForRemoval: a sync that has drifted into a duplicate-name state must still be+ // removable. See $getInfisicalSecrets for why.const payload = await $getInfisicalSecrets(secretSync, true, true);await SecretSyncFns.removeSecrets(diff --git a/backend/src/services/secret-sync/secret-sync-recursive-fns.test.ts b/backend/src/services/secret-sync/secret-sync-recursive-fns.test.tsindex 4c86073213..f616061160 100644--- a/backend/src/services/secret-sync/secret-sync-recursive-fns.test.ts+++ b/backend/src/services/secret-sync/secret-sync-recursive-fns.test.ts@@ -2,8 +2,8 @@ import {assertWithinSecretLimit,buildSyncPayload,getAncestorPaths,+ getSyncedFolders,mergeImportedSecrets,- resolveSyncFolders,SECRET_SYNC_MAX_SECRETS} from "./secret-sync-recursive-fns";@@ -15,11 +15,11 @@ const deps = {]},projectEnvDAL: { findOne: async () => ({ id: "env-1", slug: "dev", projectId: "proj-1" }) }-} as unknown as Pick<Parameters<typeof resolveSyncFolders>[0], "folderDAL" | "projectEnvDAL">;+} as unknown as Pick<Parameters<typeof getSyncedFolders>[0], "folderDAL" | "projectEnvDAL">;-describe("resolveSyncFolders", () => {+describe("getSyncedFolders", () => {test("returns only the source folder when recursive is off", async () => {- const folders = await resolveSyncFolders({+ const folders = await getSyncedFolders({...deps,projectId: "proj-1",environment: "dev",@@ -32,7 +32,7 @@ describe("resolveSyncFolders", () => {});test("returns the source folder and its descendants when recursive is on", async () => {- const folders = await resolveSyncFolders({+ const folders = await getSyncedFolders({...deps,projectId: "proj-1",environment: "dev",@@ -147,7 +147,16 @@ describe("mergeImportedSecrets", () => {expect(merged.map((entry) => entry.path).sort()).toEqual(["/backend", "/backend/api"]);});- test("a later import wins over an earlier one in the same folder, as today", () => {+ // Two import statements in the same folder resolving the same key is pre-existing, released+ // behavior: the reverse loop this reuses already shipped in secret-v2-bridge-service.ts. This+ // pins that behavior so mergeImportedSecrets doesn't change it, not because it's the right+ // design. Ideally a same-folder import collision would be a conflict error, the same way a+ // duplicate name across folders already is.+ //+ // "Later" means higher position: secretImportDAL.findByFolderIds orders imports by position+ // ascending, and that order survives unchanged through fnSecretsV2FromImports, so the import+ // listed lower in that folder's import list is what wins.+ test("the import listed later (by position) wins over an earlier one in the same folder", () => {const merged = mergeImportedSecrets([],[@@ -226,7 +235,6 @@ describe("buildSyncPayload", () => {projectFolderGrantDAL: { find: async () => [] },actorOrgId: "org-1",orgDAL: { findOrgById: async () => ({ allowCrossProjectSecretSharing: false }) },- licenseService: { getPlan: async () => ({ crossProjectSecretSharing: false }) },kmsService: {createCipherPairWithDataKey: async () => ({decryptor: () => "",@@ -236,7 +244,7 @@ describe("buildSyncPayload", () => {}} as unknown as Parameters<typeof buildSyncPayload>[0];- const payload = await buildSyncPayload(buildDeps, {+ const entries = await buildSyncPayload(buildDeps, {projectId: "proj-1",environment: "dev",sourcePath: "/",@@ -245,13 +253,19 @@ describe("buildSyncPayload", () => {includeImports: true});- const paths = payload.secrets.map((entry) => entry.path).sort();+ const paths = entries.map((entry) => entry.path).sort();expect(paths).toEqual(["/", "/api"]);});});-describe("buildSyncPayload dedupeForRemoval", () => {+// buildSyncPayload never dedupes or rejects a cross-folder name collision itself: whether that's+// a hard conflict (sync/create) or something to look past (remove, so a sync that has drifted+// into this state stays deletable) is a decision only the caller can make, made by wrapping this+// function's raw entries with createSecretSyncPayload, optionally deduping them first. See+// secret-sync-payload.test.ts for that composition, and secret-sync-queue.ts's+// $getInfisicalSecrets for the one caller that dedupes.+describe("buildSyncPayload cross-folder duplicates", () => {const duplicateNameSecret = (folderId: string, value: string) => ({id: `secret-${folderId}`,key: "DB_URL",@@ -284,7 +298,6 @@ describe("buildSyncPayload dedupeForRemoval", () => {projectFolderGrantDAL: { find: async () => [] },actorOrgId: "org-1",orgDAL: { findOrgById: async () => ({ allowCrossProjectSecretSharing: false }) },- licenseService: { getPlan: async () => ({ crossProjectSecretSharing: false }) },kmsService: {createCipherPairWithDataKey: async () => ({decryptor: () => "",@@ -303,20 +316,9 @@ describe("buildSyncPayload dedupeForRemoval", () => {includeImports: true};- // Pins the sync-path requirement from the same bug: a name used in two folders must still- // fail loudly on this path, since only the remove path may look past it.- test("a cross-folder duplicate name still throws when dedupeForRemoval is not set", async () => {- const payload = await buildSyncPayload(dedupeDeps, args);+ test("returns both colliding entries as-is, neither deduped nor rejected", async () => {+ const entries = await buildSyncPayload(dedupeDeps, args);- expect(() => payload.flatten()).toThrow(/DB_URL/);- });-- // A sync whose secrets collide across folders would otherwise be stuck: it fails to sync,- // and until this test, it also failed to remove, which is the only way to delete it.- test("dedupeForRemoval lets the remove path complete despite the same duplicate", async () => {- const payload = await buildSyncPayload(dedupeDeps, { ...args, dedupeForRemoval: true });-- expect(() => payload.flatten()).not.toThrow();- expect(Object.keys(payload.flatten())).toEqual(["DB_URL"]);+ expect(entries.map((entry) => entry.path).sort()).toEqual(["/", "/api"]);});});diff --git a/backend/src/services/secret-sync/secret-sync-recursive-fns.ts b/backend/src/services/secret-sync/secret-sync-recursive-fns.tsindex 1f1e173baf..9becc2f54a 100644--- a/backend/src/services/secret-sync/secret-sync-recursive-fns.ts+++ b/backend/src/services/secret-sync/secret-sync-recursive-fns.ts@@ -7,12 +7,7 @@ import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-foldimport { TSecretImportDALFactory } from "@app/services/secret-import/secret-import-dal";import { fnSecretsV2FromImports } from "@app/services/secret-import/secret-import-fns";import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";-import {- createSecretSyncPayload,- dedupeEntriesByDestinationKey,- TSecretPayload,- TSecretSyncPayload-} from "@app/services/secret-sync/secret-sync-payload";+import { TSecretPayload } from "@app/services/secret-sync/secret-sync-payload";import { expandSecretReferencesFactory } from "@app/services/secret-v2-bridge/secret-reference-fns";import { TSecretV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-dal";import { recursivelyGetSecretPaths } from "@app/services/secret-v2-bridge/secret-v2-bridge-fns";@@ -48,7 +43,11 @@ export const assertWithinSecretLimit = (count: number) => {});};-export const resolveSyncFolders = async ({+// A secret path is just a folder's location written as a filesystem-style string, so "folder"+// and "path" name the same node here; each folder has exactly one path and each path names+// exactly one folder, so this never returns the same folder twice. Non-recursive: just the+// source folder. Recursive: the source folder plus every folder beneath it.+export const getSyncedFolders = async ({folderDAL,projectEnvDAL,projectId,@@ -113,6 +112,13 @@ export const mergeImportedSecrets = (return merged;};+// Resolves and decrypts every secret in the sync's source subtree (a single folder, or, when+// recursive, that folder plus every folder beneath it), merging in any imports. This is the raw+// ingredient list a TSecretSyncPayload is built from; it is deliberately not itself a+// TSecretSyncPayload, since whether these entries should be deduped by destination key before+// wrapping (the remove path needs that; every other caller wants a hard conflict instead) is a+// decision specific to what each caller is about to do with them, not something this function+// should know or care about.export const buildSyncPayload = async (deps: {folderDAL: Pick<TSecretFolderDALFactory, "find" | "findByManySecretPath">;@@ -129,19 +135,14 @@ export const buildSyncPayload = async (sourcePath: string;sourceFolderId: string;recursive: boolean;- keySchema?: string;includeImports: boolean;- // The remove path passes true so a duplicate name across folders cannot leave the sync- // undeletable; every other caller must keep the duplicate-name check intact.- dedupeForRemoval?: boolean;}-): Promise<TSecretSyncPayload> => {+): Promise<TSecretPayload[]> => {const { folderDAL, projectEnvDAL, secretV2BridgeDAL, secretImportDAL, expandSecretReferences, decryptSecretValue } =deps;- const { projectId, environment, sourcePath, sourceFolderId, recursive, keySchema, includeImports, dedupeForRemoval } =- args;+ const { projectId, environment, sourcePath, sourceFolderId, recursive, includeImports } = args;- const folders = await resolveSyncFolders({+ const folders = await getSyncedFolders({folderDAL,projectEnvDAL,projectId,@@ -194,10 +195,7 @@ export const buildSyncPayload = async ();if (!includeImports) {- return createSecretSyncPayload(- dedupeForRemoval ? dedupeEntriesByDestinationKey(entries, { environment, keySchema }) : entries,- { environment, keySchema }- );+ return entries;}const secretImports = await secretImportDAL.findByFolderIds(folders.map(({ folderId }) => folderId));@@ -242,8 +240,5 @@ export const buildSyncPayload = async (assertWithinSecretLimit(allEntries.length);}- return createSecretSyncPayload(- dedupeForRemoval ? dedupeEntriesByDestinationKey(allEntries, { environment, keySchema }) : allEntries,- { environment, keySchema }- );+ return allEntries;};diff --git a/backend/src/services/secret-sync/secret-sync-schemas.test.ts b/backend/src/services/secret-sync/secret-sync-schemas.test.tsindex a8948f39f3..156b1bd55b 100644--- a/backend/src/services/secret-sync/secret-sync-schemas.test.ts+++ b/backend/src/services/secret-sync/secret-sync-schemas.test.ts@@ -20,6 +20,9 @@ describe("BaseSyncOptionsSchema recursive", () => {expect(result.success).toBe(true);});+ // Importing would need to write each secret back into the subfolder it came from, which isn't+ // supported yet, so allowing this combination would squash every subfolder's secrets into the+ // sync's root folder instead. See RECURSIVE_SYNC_REFINEMENT.test("rejects recursive combined with importing from the destination", () => {const result = schema.safeParse({initialSyncBehavior: SecretSyncInitialSyncBehavior.ImportPrioritizeSource,diff --git a/backend/src/services/secret-sync/secret-sync-schemas.ts b/backend/src/services/secret-sync/secret-sync-schemas.tsindex b2fbc0966b..7bbd35b5c0 100644--- a/backend/src/services/secret-sync/secret-sync-schemas.ts+++ b/backend/src/services/secret-sync/secret-sync-schemas.ts@@ -9,6 +9,10 @@ import { SecretSync, SecretSyncInitialSyncBehavior } from "@app/services/secret-import { SECRET_SYNC_CONNECTION_MAP, SECRET_SYNC_NAME_MAP } from "@app/services/secret-sync/secret-sync-maps";import { TSyncOptionsConfig } from "@app/services/secret-sync/secret-sync-types";+// Importing would need to write each secret back into the subfolder it came from, which we don't+// support yet (hierarchical writes to destinations that can represent folders are planned for a+// later PR). Until then, allowing this combination would silently squash every subfolder's+// secrets into the sync's root folder, restructuring the user's existing secrets tree.const RECURSIVE_SYNC_REFINEMENT = {path: ["recursive"],message:@@ -76,15 +80,18 @@ const BaseSyncOptionsSchema = <T extends AnyZodObject | undefined = undefined>({recursive: z.boolean().optional().describe(SecretSyncs.SYNC_OPTIONS(destination).recursive)});+ // What refinedSchema actually is: baseSchema, merged with the destination's own extra+ // sync-option fields when `merge` supplies them, wrapped in a Zod effect that enforces+ // isRecursiveCombinationAllowed across the result.type TRefinedSchema = z.ZodEffects<T extends AnyZodObject? z.ZodObject<z.objectUtil.MergeShapes<typeof baseSchema.shape, T["shape"]>>: typeof baseSchema>;- // `merge` is supplied exactly when `T` is a ZodObject, which is the condition this type branches- // on, and the compiler cannot correlate a runtime ternary with a type parameter. Passing an- // explicit type argument that disagrees with the `merge` argument would make the assertion false.+ // The cast below is needed because TypeScript can't verify it on its own: every caller passes+ // `merge` and a real T together or neither, so the runtime ternary and the type-level ternary+ // above always agree, but the compiler has no way to prove that from a runtime value.const refinedSchema = (merge? baseSchema.merge(merge).refine(isRecursiveCombinationAllowed, RECURSIVE_SYNC_REFINEMENT)@@ -170,5 +177,10 @@ export const GenericUpdateSecretSyncFieldsSchema = <T extends AnyZodObject | und.optional().describe(SecretSyncs.UPDATE(destination).secretPath),isAutoSyncEnabled: z.boolean().optional().describe(SecretSyncs.UPDATE(destination).isAutoSyncEnabled),+ // Optional like every other field here, so an update can touch just the name or description+ // without resending sync options. This can't be used to sneak past+ // isRecursiveCombinationAllowed: initialSyncBehavior inside BaseSyncOptionsSchema stays+ // required, so a request that includes syncOptions at all must still submit a complete,+ // valid one.syncOptions: BaseSyncOptionsSchema({ destination, syncOptionsConfig, merge, isUpdateSchema: true }).optional()});diff --git a/backend/src/services/secret-sync/secret-sync-service.ts b/backend/src/services/secret-sync/secret-sync-service.tsindex 66e3830e05..5f0ec4e801 100644--- a/backend/src/services/secret-sync/secret-sync-service.ts+++ b/backend/src/services/secret-sync/secret-sync-service.ts@@ -34,7 +34,8 @@ import {preSaveTransformDestinationConfig,preSaveTransformSyncOptions} from "@app/services/secret-sync/secret-sync-fns";-import { buildSyncPayload, resolveSyncFolders } from "@app/services/secret-sync/secret-sync-recursive-fns";+import { createSecretSyncPayload } from "@app/services/secret-sync/secret-sync-payload";+import { buildSyncPayload, getSyncedFolders } from "@app/services/secret-sync/secret-sync-recursive-fns";import {SecretSyncStatus,TCheckDuplicateDestinationDTO,@@ -162,7 +163,7 @@ export const secretSyncServiceFactory = ({recursive: boolean;}) => {- const folders = await resolveSyncFolders({+ const folders = await getSyncedFolders({folderDAL,projectEnvDAL,projectId,@@ -247,7 +248,7 @@ export const secretSyncServiceFactory = ({kmsService});- return buildSyncPayload(+ const entries = await buildSyncPayload({folderDAL,projectEnvDAL,@@ -268,11 +269,11 @@ export const secretSyncServiceFactory = ({sourcePath,sourceFolderId,recursive: true,- keySchema,- includeImports: true,- dedupeForRemoval: false+ includeImports: true});++ return createSecretSyncPayload(entries, { environment, keySchema });});// Flattening a subtree onto a destination that holds one flat list can produce two secrets with