Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.
TypeScript acmeagentsai-agentscertificate-managementdevsecopsenvironment-variableskubernetesopen-sourcepampkipostgresprivate-caprivileged-access-managementsecret-managementsecret-managersecretssecrets-managementsecuritysecurity-toolsvault
refactor(secret-sync): make dedupeConflicts a payload method instead of a builder flag
Calvin6ee9f1fdcac1
diff --git a/backend/src/services/secret-sync/secret-sync-payload.test.ts b/backend/src/services/secret-sync/secret-sync-payload.test.tsindex ee86fb691a..b7605950e5 100644--- a/backend/src/services/secret-sync/secret-sync-payload.test.ts+++ b/backend/src/services/secret-sync/secret-sync-payload.test.ts@@ -112,6 +112,25 @@ describe("createSecretSyncPayload", () => {{ key: "DB_URL", paths: ["/backend", "/backend/api"] }]);});++ test("dedupeConflicts() returns a payload whose flatten() no longer throws", () => {+ const payload = createSecretSyncPayload([secret("DB_URL", "/backend"), secret("DB_URL", "/backend/api")], {+ environment: "dev"+ });++ const deduped = payload.dedupeConflicts();++ expect(() => deduped.flatten()).not.toThrow();+ expect(Object.keys(deduped.flatten())).toEqual(["DB_URL"]);+ });++ test("dedupeConflicts() leaves a payload with no conflicts unchanged", () => {+ const payload = createSecretSyncPayload([secret("DB_URL", "/"), secret("API_KEY", "/api")], {+ environment: "dev"+ });++ expect(payload.dedupeConflicts().secrets).toHaveLength(2);+ });});describe("findFlattenConflicts", () => {diff --git a/backend/src/services/secret-sync/secret-sync-payload.ts b/backend/src/services/secret-sync/secret-sync-payload.tsindex 64e5efd61d..e5932eed44 100644--- a/backend/src/services/secret-sync/secret-sync-payload.ts+++ b/backend/src/services/secret-sync/secret-sync-payload.ts@@ -19,6 +19,7 @@ export type TSecretSyncPayload = {environment: string;flatten: (opts?: { applySchema?: boolean }) => TSecretMap;findConflicts: (opts?: { applySchema?: boolean }) => TSecretSyncConflict[];+ dedupeConflicts: () => TSecretSyncPayload;};export type TSecretSyncConflict = {@@ -90,10 +91,10 @@ const buildConflictError = (conflicts: TSecretSyncConflict[]) => {});};-// The remove path deletes by destination key and does not care which duplicate wins, so it-// dedupes before the payload is built. That keeps flatten() itself unconditional: a sync that-// has drifted into a duplicate-name state must still be removable, since deleteSyncOnComplete-// only drops the sync row after a successful remove.+// Removal deletes by destination key and does not care which duplicate wins, so dedupeConflicts()+// (below) uses this to resolve a conflict before a provider's own flatten() call ever sees it: a+// sync that has drifted into a duplicate-name state must still be removable, since+// deleteSyncOnComplete only drops the sync row after a successful remove.export const dedupeEntriesByDestinationKey = (entries: TSecretPayload[],{ environment, keySchema }: { environment: string; keySchema?: string }@@ -151,5 +152,14 @@ export const createSecretSyncPayload = (// keySchema itself is deliberately not a field on this object: nothing outside this module// needs to read it directly, and findConflicts() is how an external caller (the// recursive-conflicts preview endpoint) reaches the same check flatten() runs, without it.- findConflicts: (opts) => findFlattenConflicts({ secrets, environment, keySchema }, opts)+ findConflicts: (opts) => findFlattenConflicts({ secrets, environment, keySchema }, opts),+ // Removal hands the whole payload to a provider's own removeSecrets, which calls flatten()+ // itself with no idea this is a removal, so tolerating a duplicate name has to happen here,+ // before that flatten() call exists to throw on it. Returns a new payload; this one's+ // duplicate-free by construction, so its own flatten() call, whatever calls it, never conflicts.+ dedupeConflicts: () =>+ createSecretSyncPayload(dedupeEntriesByDestinationKey(secrets, { environment, keySchema }), {+ environment,+ keySchema+ })});diff --git a/backend/src/services/secret-sync/secret-sync-queue.ts b/backend/src/services/secret-sync/secret-sync-queue.tsindex c5b6eca4c5..e30dd39f22 100644--- a/backend/src/services/secret-sync/secret-sync-queue.ts+++ b/backend/src/services/secret-sync/secret-sync-queue.ts@@ -44,7 +44,6 @@ import {import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";import { enterpriseSyncCheck, parseSyncErrorMessage, SecretSyncFns } from "@app/services/secret-sync/secret-sync-fns";import { SECRET_SYNC_DAILY_RETRY_DESTINATIONS, SECRET_SYNC_NAME_MAP } from "@app/services/secret-sync/secret-sync-maps";-import { createSecretSyncPayload, dedupeEntriesByDestinationKey } from "@app/services/secret-sync/secret-sync-payload";import { buildSyncPayload, getAncestorPaths } from "@app/services/secret-sync/secret-sync-recursive-fns";import {SecretSyncAction,@@ -268,8 +267,7 @@ export const secretSyncQueueFactory = ({const $getInfisicalSecrets = async (secretSync: TSecretSyncRaw | TSecretSyncWithCredentials,- includeImports = true,- dedupeForRemoval = false+ includeImports = true) => {const { projectId, folderId, environment, folder } = secretSync;@@ -303,7 +301,7 @@ export const secretSyncQueueFactory = ({kmsService});- const entries = await buildSyncPayload(+ return buildSyncPayload({folderDAL,projectEnvDAL,@@ -324,19 +322,10 @@ export const secretSyncQueueFactory = ({sourcePath: folder.path,sourceFolderId: folderId,recursive: Boolean(syncOptions?.recursive),+ keySchema: syncOptions?.keySchema,includeImports});-- // Duplicates across folders are deduped before the payload is built, so flatten() cannot- // throw here: a sync that has drifted into a duplicate-name state must still be removable.- // Every other caller keeps the duplicate-name check intact.- return createSecretSyncPayload(- dedupeForRemoval- ? dedupeEntriesByDestinationKey(entries, { environment: environment.slug, keySchema: syncOptions?.keySchema })- : entries,- { environment: environment.slug, keySchema: syncOptions?.keySchema }- );};const queueSecretSyncSyncSecretsById = async (payload: TQueueSecretSyncSyncSecretsByIdDTO) => {@@ -840,9 +829,7 @@ export const secretSyncQueueFactory = ({projectId});- // dedupeForRemoval: a sync that has drifted into a duplicate-name state must still be- // removable. See $getInfisicalSecrets for why.- const payload = await $getInfisicalSecrets(secretSync, true, true);+ const payload = await $getInfisicalSecrets(secretSync, true);await SecretSyncFns.removeSecrets({@@ -852,7 +839,9 @@ export const secretSyncQueueFactory = ({credentials}} as TSecretSyncWithCredentials,- payload,+ // A sync that has drifted into a duplicate-name state must still be removable: dedupe+ // before any provider's own flatten() call gets a chance to throw on it.+ payload.dedupeConflicts(),{appConnectionDAL,gitHubAppDAL,diff --git a/backend/src/services/secret-sync/secret-sync-recursive-fns.test.ts b/backend/src/services/secret-sync/secret-sync-recursive-fns.test.tsindex f616061160..cadd58c755 100644--- a/backend/src/services/secret-sync/secret-sync-recursive-fns.test.ts+++ b/backend/src/services/secret-sync/secret-sync-recursive-fns.test.ts@@ -244,7 +244,7 @@ describe("buildSyncPayload", () => {}} as unknown as Parameters<typeof buildSyncPayload>[0];- const entries = await buildSyncPayload(buildDeps, {+ const payload = await buildSyncPayload(buildDeps, {projectId: "proj-1",environment: "dev",sourcePath: "/",@@ -253,18 +253,12 @@ describe("buildSyncPayload", () => {includeImports: true});- const paths = entries.map((entry) => entry.path).sort();+ const paths = payload.secrets.map((entry) => entry.path).sort();expect(paths).toEqual(["/", "/api"]);});});-// buildSyncPayload never dedupes or rejects a cross-folder name collision itself: whether that's-// a hard conflict (sync/create) or something to look past (remove, so a sync that has drifted-// into this state stays deletable) is a decision only the caller can make, made by wrapping this-// function's raw entries with createSecretSyncPayload, optionally deduping them first. See-// secret-sync-payload.test.ts for that composition, and secret-sync-queue.ts's-// $getInfisicalSecrets for the one caller that dedupes.describe("buildSyncPayload cross-folder duplicates", () => {const duplicateNameSecret = (folderId: string, value: string) => ({id: `secret-${folderId}`,@@ -316,9 +310,21 @@ describe("buildSyncPayload cross-folder duplicates", () => {includeImports: true};- test("returns both colliding entries as-is, neither deduped nor rejected", async () => {- const entries = await buildSyncPayload(dedupeDeps, args);+ // Pins the sync-path requirement from the same bug: a name used in two folders must still fail+ // loudly on this path, since only the remove path may look past it.+ test("a cross-folder duplicate name still throws by default", async () => {+ const payload = await buildSyncPayload(dedupeDeps, args);- expect(entries.map((entry) => entry.path).sort()).toEqual(["/", "/api"]);+ expect(() => payload.flatten()).toThrow(/DB_URL/);+ });++ // A sync whose secrets collide across folders would otherwise be stuck: it fails to sync, and+ // until dedupeConflicts() existed, it also failed to remove, which is the only way to delete it.+ test("dedupeConflicts() lets the remove path complete despite the same duplicate", async () => {+ const payload = await buildSyncPayload(dedupeDeps, args);+ const deduped = payload.dedupeConflicts();++ expect(() => deduped.flatten()).not.toThrow();+ expect(Object.keys(deduped.flatten())).toEqual(["DB_URL"]);});});diff --git a/backend/src/services/secret-sync/secret-sync-recursive-fns.ts b/backend/src/services/secret-sync/secret-sync-recursive-fns.tsindex 9becc2f54a..d6177a08bf 100644--- a/backend/src/services/secret-sync/secret-sync-recursive-fns.ts+++ b/backend/src/services/secret-sync/secret-sync-recursive-fns.ts@@ -7,7 +7,11 @@ import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-foldimport { TSecretImportDALFactory } from "@app/services/secret-import/secret-import-dal";import { fnSecretsV2FromImports } from "@app/services/secret-import/secret-import-fns";import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";-import { TSecretPayload } from "@app/services/secret-sync/secret-sync-payload";+import {+ createSecretSyncPayload,+ TSecretPayload,+ TSecretSyncPayload+} from "@app/services/secret-sync/secret-sync-payload";import { expandSecretReferencesFactory } from "@app/services/secret-v2-bridge/secret-reference-fns";import { TSecretV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-dal";import { recursivelyGetSecretPaths } from "@app/services/secret-v2-bridge/secret-v2-bridge-fns";@@ -113,12 +117,10 @@ export const mergeImportedSecrets = (};// Resolves and decrypts every secret in the sync's source subtree (a single folder, or, when-// recursive, that folder plus every folder beneath it), merging in any imports. This is the raw-// ingredient list a TSecretSyncPayload is built from; it is deliberately not itself a-// TSecretSyncPayload, since whether these entries should be deduped by destination key before-// wrapping (the remove path needs that; every other caller wants a hard conflict instead) is a-// decision specific to what each caller is about to do with them, not something this function-// should know or care about.+// recursive, that folder plus every folder beneath it), merging in any imports, and wraps the+// result in a TSecretSyncPayload. A cross-folder duplicate name is not resolved here: flatten()+// throws on it by default, and payload.dedupeConflicts() is how the one caller that needs to+// tolerate it (removal) gets a payload that won't.export const buildSyncPayload = async (deps: {folderDAL: Pick<TSecretFolderDALFactory, "find" | "findByManySecretPath">;@@ -135,12 +137,13 @@ export const buildSyncPayload = async (sourcePath: string;sourceFolderId: string;recursive: boolean;+ keySchema?: string;includeImports: boolean;}-): Promise<TSecretPayload[]> => {+): Promise<TSecretSyncPayload> => {const { folderDAL, projectEnvDAL, secretV2BridgeDAL, secretImportDAL, expandSecretReferences, decryptSecretValue } =deps;- const { projectId, environment, sourcePath, sourceFolderId, recursive, includeImports } = args;+ const { projectId, environment, sourcePath, sourceFolderId, recursive, keySchema, includeImports } = args;const folders = await getSyncedFolders({folderDAL,@@ -195,7 +198,7 @@ export const buildSyncPayload = async ();if (!includeImports) {- return entries;+ return createSecretSyncPayload(entries, { environment, keySchema });}const secretImports = await secretImportDAL.findByFolderIds(folders.map(({ folderId }) => folderId));@@ -240,5 +243,5 @@ export const buildSyncPayload = async (assertWithinSecretLimit(allEntries.length);}- return allEntries;+ return createSecretSyncPayload(allEntries, { environment, keySchema });};diff --git a/backend/src/services/secret-sync/secret-sync-service.ts b/backend/src/services/secret-sync/secret-sync-service.tsindex 5f0ec4e801..a130fb91c5 100644--- a/backend/src/services/secret-sync/secret-sync-service.ts+++ b/backend/src/services/secret-sync/secret-sync-service.ts@@ -34,7 +34,6 @@ import {preSaveTransformDestinationConfig,preSaveTransformSyncOptions} from "@app/services/secret-sync/secret-sync-fns";-import { createSecretSyncPayload } from "@app/services/secret-sync/secret-sync-payload";import { buildSyncPayload, getSyncedFolders } from "@app/services/secret-sync/secret-sync-recursive-fns";import {SecretSyncStatus,@@ -248,7 +247,7 @@ export const secretSyncServiceFactory = ({kmsService});- const entries = await buildSyncPayload(+ return buildSyncPayload({folderDAL,projectEnvDAL,@@ -269,11 +268,10 @@ export const secretSyncServiceFactory = ({sourcePath,sourceFolderId,recursive: true,+ keySchema,includeImports: true});-- return createSecretSyncPayload(entries, { environment, keySchema });});// Flattening a subtree onto a destination that holds one flat list can produce two secrets with