Skip to content

Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.

refactor(secret-sync): make dedupeConflicts a payload method instead of a builder flag

Calvin6ee9f1fdcac1
diff --git a/backend/src/services/secret-sync/secret-sync-payload.test.ts b/backend/src/services/secret-sync/secret-sync-payload.test.ts
index ee86fb691a..b7605950e5 100644
--- a/backend/src/services/secret-sync/secret-sync-payload.test.ts
+++ b/backend/src/services/secret-sync/secret-sync-payload.test.ts
@@ -112,6 +112,25 @@ describe("createSecretSyncPayload", () => {
{ key: "DB_URL", paths: ["/backend", "/backend/api"] }
]);
});
+
+ test("dedupeConflicts() returns a payload whose flatten() no longer throws", () => {
+ const payload = createSecretSyncPayload([secret("DB_URL", "/backend"), secret("DB_URL", "/backend/api")], {
+ environment: "dev"
+ });
+
+ const deduped = payload.dedupeConflicts();
+
+ expect(() => deduped.flatten()).not.toThrow();
+ expect(Object.keys(deduped.flatten())).toEqual(["DB_URL"]);
+ });
+
+ test("dedupeConflicts() leaves a payload with no conflicts unchanged", () => {
+ const payload = createSecretSyncPayload([secret("DB_URL", "/"), secret("API_KEY", "/api")], {
+ environment: "dev"
+ });
+
+ expect(payload.dedupeConflicts().secrets).toHaveLength(2);
+ });
});
describe("findFlattenConflicts", () => {
diff --git a/backend/src/services/secret-sync/secret-sync-payload.ts b/backend/src/services/secret-sync/secret-sync-payload.ts
index 64e5efd61d..e5932eed44 100644
--- a/backend/src/services/secret-sync/secret-sync-payload.ts
+++ b/backend/src/services/secret-sync/secret-sync-payload.ts
@@ -19,6 +19,7 @@ export type TSecretSyncPayload = {
environment: string;
flatten: (opts?: { applySchema?: boolean }) => TSecretMap;
findConflicts: (opts?: { applySchema?: boolean }) => TSecretSyncConflict[];
+ dedupeConflicts: () => TSecretSyncPayload;
};
export type TSecretSyncConflict = {
@@ -90,10 +91,10 @@ const buildConflictError = (conflicts: TSecretSyncConflict[]) => {
});
};
-// The remove path deletes by destination key and does not care which duplicate wins, so it
-// dedupes before the payload is built. That keeps flatten() itself unconditional: a sync that
-// has drifted into a duplicate-name state must still be removable, since deleteSyncOnComplete
-// only drops the sync row after a successful remove.
+// Removal deletes by destination key and does not care which duplicate wins, so dedupeConflicts()
+// (below) uses this to resolve a conflict before a provider's own flatten() call ever sees it: a
+// sync that has drifted into a duplicate-name state must still be removable, since
+// deleteSyncOnComplete only drops the sync row after a successful remove.
export const dedupeEntriesByDestinationKey = (
entries: TSecretPayload[],
{ environment, keySchema }: { environment: string; keySchema?: string }
@@ -151,5 +152,14 @@ export const createSecretSyncPayload = (
// keySchema itself is deliberately not a field on this object: nothing outside this module
// needs to read it directly, and findConflicts() is how an external caller (the
// recursive-conflicts preview endpoint) reaches the same check flatten() runs, without it.
- findConflicts: (opts) => findFlattenConflicts({ secrets, environment, keySchema }, opts)
+ findConflicts: (opts) => findFlattenConflicts({ secrets, environment, keySchema }, opts),
+ // Removal hands the whole payload to a provider's own removeSecrets, which calls flatten()
+ // itself with no idea this is a removal, so tolerating a duplicate name has to happen here,
+ // before that flatten() call exists to throw on it. Returns a new payload; this one's
+ // duplicate-free by construction, so its own flatten() call, whatever calls it, never conflicts.
+ dedupeConflicts: () =>
+ createSecretSyncPayload(dedupeEntriesByDestinationKey(secrets, { environment, keySchema }), {
+ environment,
+ keySchema
+ })
});
diff --git a/backend/src/services/secret-sync/secret-sync-queue.ts b/backend/src/services/secret-sync/secret-sync-queue.ts
index c5b6eca4c5..e30dd39f22 100644
--- a/backend/src/services/secret-sync/secret-sync-queue.ts
+++ b/backend/src/services/secret-sync/secret-sync-queue.ts
@@ -44,7 +44,6 @@ import {
import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";
import { enterpriseSyncCheck, parseSyncErrorMessage, SecretSyncFns } from "@app/services/secret-sync/secret-sync-fns";
import { SECRET_SYNC_DAILY_RETRY_DESTINATIONS, SECRET_SYNC_NAME_MAP } from "@app/services/secret-sync/secret-sync-maps";
-import { createSecretSyncPayload, dedupeEntriesByDestinationKey } from "@app/services/secret-sync/secret-sync-payload";
import { buildSyncPayload, getAncestorPaths } from "@app/services/secret-sync/secret-sync-recursive-fns";
import {
SecretSyncAction,
@@ -268,8 +267,7 @@ export const secretSyncQueueFactory = ({
const $getInfisicalSecrets = async (
secretSync: TSecretSyncRaw | TSecretSyncWithCredentials,
- includeImports = true,
- dedupeForRemoval = false
+ includeImports = true
) => {
const { projectId, folderId, environment, folder } = secretSync;
@@ -303,7 +301,7 @@ export const secretSyncQueueFactory = ({
kmsService
});
- const entries = await buildSyncPayload(
+ return buildSyncPayload(
{
folderDAL,
projectEnvDAL,
@@ -324,19 +322,10 @@ export const secretSyncQueueFactory = ({
sourcePath: folder.path,
sourceFolderId: folderId,
recursive: Boolean(syncOptions?.recursive),
+ keySchema: syncOptions?.keySchema,
includeImports
}
);
-
- // Duplicates across folders are deduped before the payload is built, so flatten() cannot
- // throw here: a sync that has drifted into a duplicate-name state must still be removable.
- // Every other caller keeps the duplicate-name check intact.
- return createSecretSyncPayload(
- dedupeForRemoval
- ? dedupeEntriesByDestinationKey(entries, { environment: environment.slug, keySchema: syncOptions?.keySchema })
- : entries,
- { environment: environment.slug, keySchema: syncOptions?.keySchema }
- );
};
const queueSecretSyncSyncSecretsById = async (payload: TQueueSecretSyncSyncSecretsByIdDTO) => {
@@ -840,9 +829,7 @@ export const secretSyncQueueFactory = ({
projectId
});
- // dedupeForRemoval: a sync that has drifted into a duplicate-name state must still be
- // removable. See $getInfisicalSecrets for why.
- const payload = await $getInfisicalSecrets(secretSync, true, true);
+ const payload = await $getInfisicalSecrets(secretSync, true);
await SecretSyncFns.removeSecrets(
{
@@ -852,7 +839,9 @@ export const secretSyncQueueFactory = ({
credentials
}
} as TSecretSyncWithCredentials,
- payload,
+ // A sync that has drifted into a duplicate-name state must still be removable: dedupe
+ // before any provider's own flatten() call gets a chance to throw on it.
+ payload.dedupeConflicts(),
{
appConnectionDAL,
gitHubAppDAL,
diff --git a/backend/src/services/secret-sync/secret-sync-recursive-fns.test.ts b/backend/src/services/secret-sync/secret-sync-recursive-fns.test.ts
index f616061160..cadd58c755 100644
--- a/backend/src/services/secret-sync/secret-sync-recursive-fns.test.ts
+++ b/backend/src/services/secret-sync/secret-sync-recursive-fns.test.ts
@@ -244,7 +244,7 @@ describe("buildSyncPayload", () => {
}
} as unknown as Parameters<typeof buildSyncPayload>[0];
- const entries = await buildSyncPayload(buildDeps, {
+ const payload = await buildSyncPayload(buildDeps, {
projectId: "proj-1",
environment: "dev",
sourcePath: "/",
@@ -253,18 +253,12 @@ describe("buildSyncPayload", () => {
includeImports: true
});
- const paths = entries.map((entry) => entry.path).sort();
+ const paths = payload.secrets.map((entry) => entry.path).sort();
expect(paths).toEqual(["/", "/api"]);
});
});
-// buildSyncPayload never dedupes or rejects a cross-folder name collision itself: whether that's
-// a hard conflict (sync/create) or something to look past (remove, so a sync that has drifted
-// into this state stays deletable) is a decision only the caller can make, made by wrapping this
-// function's raw entries with createSecretSyncPayload, optionally deduping them first. See
-// secret-sync-payload.test.ts for that composition, and secret-sync-queue.ts's
-// $getInfisicalSecrets for the one caller that dedupes.
describe("buildSyncPayload cross-folder duplicates", () => {
const duplicateNameSecret = (folderId: string, value: string) => ({
id: `secret-${folderId}`,
@@ -316,9 +310,21 @@ describe("buildSyncPayload cross-folder duplicates", () => {
includeImports: true
};
- test("returns both colliding entries as-is, neither deduped nor rejected", async () => {
- const entries = await buildSyncPayload(dedupeDeps, args);
+ // Pins the sync-path requirement from the same bug: a name used in two folders must still fail
+ // loudly on this path, since only the remove path may look past it.
+ test("a cross-folder duplicate name still throws by default", async () => {
+ const payload = await buildSyncPayload(dedupeDeps, args);
- expect(entries.map((entry) => entry.path).sort()).toEqual(["/", "/api"]);
+ expect(() => payload.flatten()).toThrow(/DB_URL/);
+ });
+
+ // A sync whose secrets collide across folders would otherwise be stuck: it fails to sync, and
+ // until dedupeConflicts() existed, it also failed to remove, which is the only way to delete it.
+ test("dedupeConflicts() lets the remove path complete despite the same duplicate", async () => {
+ const payload = await buildSyncPayload(dedupeDeps, args);
+ const deduped = payload.dedupeConflicts();
+
+ expect(() => deduped.flatten()).not.toThrow();
+ expect(Object.keys(deduped.flatten())).toEqual(["DB_URL"]);
});
});
diff --git a/backend/src/services/secret-sync/secret-sync-recursive-fns.ts b/backend/src/services/secret-sync/secret-sync-recursive-fns.ts
index 9becc2f54a..d6177a08bf 100644
--- a/backend/src/services/secret-sync/secret-sync-recursive-fns.ts
+++ b/backend/src/services/secret-sync/secret-sync-recursive-fns.ts
@@ -7,7 +7,11 @@ import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-fold
import { TSecretImportDALFactory } from "@app/services/secret-import/secret-import-dal";
import { fnSecretsV2FromImports } from "@app/services/secret-import/secret-import-fns";
import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";
-import { TSecretPayload } from "@app/services/secret-sync/secret-sync-payload";
+import {
+ createSecretSyncPayload,
+ TSecretPayload,
+ TSecretSyncPayload
+} from "@app/services/secret-sync/secret-sync-payload";
import { expandSecretReferencesFactory } from "@app/services/secret-v2-bridge/secret-reference-fns";
import { TSecretV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-dal";
import { recursivelyGetSecretPaths } from "@app/services/secret-v2-bridge/secret-v2-bridge-fns";
@@ -113,12 +117,10 @@ export const mergeImportedSecrets = (
};
// Resolves and decrypts every secret in the sync's source subtree (a single folder, or, when
-// recursive, that folder plus every folder beneath it), merging in any imports. This is the raw
-// ingredient list a TSecretSyncPayload is built from; it is deliberately not itself a
-// TSecretSyncPayload, since whether these entries should be deduped by destination key before
-// wrapping (the remove path needs that; every other caller wants a hard conflict instead) is a
-// decision specific to what each caller is about to do with them, not something this function
-// should know or care about.
+// recursive, that folder plus every folder beneath it), merging in any imports, and wraps the
+// result in a TSecretSyncPayload. A cross-folder duplicate name is not resolved here: flatten()
+// throws on it by default, and payload.dedupeConflicts() is how the one caller that needs to
+// tolerate it (removal) gets a payload that won't.
export const buildSyncPayload = async (
deps: {
folderDAL: Pick<TSecretFolderDALFactory, "find" | "findByManySecretPath">;
@@ -135,12 +137,13 @@ export const buildSyncPayload = async (
sourcePath: string;
sourceFolderId: string;
recursive: boolean;
+ keySchema?: string;
includeImports: boolean;
}
-): Promise<TSecretPayload[]> => {
+): Promise<TSecretSyncPayload> => {
const { folderDAL, projectEnvDAL, secretV2BridgeDAL, secretImportDAL, expandSecretReferences, decryptSecretValue } =
deps;
- const { projectId, environment, sourcePath, sourceFolderId, recursive, includeImports } = args;
+ const { projectId, environment, sourcePath, sourceFolderId, recursive, keySchema, includeImports } = args;
const folders = await getSyncedFolders({
folderDAL,
@@ -195,7 +198,7 @@ export const buildSyncPayload = async (
);
if (!includeImports) {
- return entries;
+ return createSecretSyncPayload(entries, { environment, keySchema });
}
const secretImports = await secretImportDAL.findByFolderIds(folders.map(({ folderId }) => folderId));
@@ -240,5 +243,5 @@ export const buildSyncPayload = async (
assertWithinSecretLimit(allEntries.length);
}
- return allEntries;
+ return createSecretSyncPayload(allEntries, { environment, keySchema });
};
diff --git a/backend/src/services/secret-sync/secret-sync-service.ts b/backend/src/services/secret-sync/secret-sync-service.ts
index 5f0ec4e801..a130fb91c5 100644
--- a/backend/src/services/secret-sync/secret-sync-service.ts
+++ b/backend/src/services/secret-sync/secret-sync-service.ts
@@ -34,7 +34,6 @@ import {
preSaveTransformDestinationConfig,
preSaveTransformSyncOptions
} from "@app/services/secret-sync/secret-sync-fns";
-import { createSecretSyncPayload } from "@app/services/secret-sync/secret-sync-payload";
import { buildSyncPayload, getSyncedFolders } from "@app/services/secret-sync/secret-sync-recursive-fns";
import {
SecretSyncStatus,
@@ -248,7 +247,7 @@ export const secretSyncServiceFactory = ({
kmsService
});
- const entries = await buildSyncPayload(
+ return buildSyncPayload(
{
folderDAL,
projectEnvDAL,
@@ -269,11 +268,10 @@ export const secretSyncServiceFactory = ({
sourcePath,
sourceFolderId,
recursive: true,
+ keySchema,
includeImports: true
}
);
-
- return createSecretSyncPayload(entries, { environment, keySchema });
});
// Flattening a subtree onto a destination that holds one flat list can produce two secrets with