Skip to content

Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.

fix(secret-sync): address review comments on buildSyncPayload and schema docs

Calvin703c733a3406
diff --git a/backend/src/services/secret-sync/secret-sync-queue.ts b/backend/src/services/secret-sync/secret-sync-queue.ts
index ee233665e3..c5b6eca4c5 100644
--- a/backend/src/services/secret-sync/secret-sync-queue.ts
+++ b/backend/src/services/secret-sync/secret-sync-queue.ts
@@ -44,6 +44,7 @@ import {
import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";
import { enterpriseSyncCheck, parseSyncErrorMessage, SecretSyncFns } from "@app/services/secret-sync/secret-sync-fns";
import { SECRET_SYNC_DAILY_RETRY_DESTINATIONS, SECRET_SYNC_NAME_MAP } from "@app/services/secret-sync/secret-sync-maps";
+import { createSecretSyncPayload, dedupeEntriesByDestinationKey } from "@app/services/secret-sync/secret-sync-payload";
import { buildSyncPayload, getAncestorPaths } from "@app/services/secret-sync/secret-sync-recursive-fns";
import {
SecretSyncAction,
@@ -302,7 +303,7 @@ export const secretSyncQueueFactory = ({
kmsService
});
- return buildSyncPayload(
+ const entries = await buildSyncPayload(
{
folderDAL,
projectEnvDAL,
@@ -323,11 +324,19 @@ export const secretSyncQueueFactory = ({
sourcePath: folder.path,
sourceFolderId: folderId,
recursive: Boolean(syncOptions?.recursive),
- keySchema: syncOptions?.keySchema,
- includeImports,
- dedupeForRemoval
+ includeImports
}
);
+
+ // Duplicates across folders are deduped before the payload is built, so flatten() cannot
+ // throw here: a sync that has drifted into a duplicate-name state must still be removable.
+ // Every other caller keeps the duplicate-name check intact.
+ return createSecretSyncPayload(
+ dedupeForRemoval
+ ? dedupeEntriesByDestinationKey(entries, { environment: environment.slug, keySchema: syncOptions?.keySchema })
+ : entries,
+ { environment: environment.slug, keySchema: syncOptions?.keySchema }
+ );
};
const queueSecretSyncSyncSecretsById = async (payload: TQueueSecretSyncSyncSecretsByIdDTO) => {
@@ -831,8 +840,8 @@ export const secretSyncQueueFactory = ({
projectId
});
- // Duplicates across folders are deduped before the payload is built, so flatten() cannot
- // throw here: a sync that has drifted into a duplicate-name state must still be removable.
+ // dedupeForRemoval: a sync that has drifted into a duplicate-name state must still be
+ // removable. See $getInfisicalSecrets for why.
const payload = await $getInfisicalSecrets(secretSync, true, true);
await SecretSyncFns.removeSecrets(
diff --git a/backend/src/services/secret-sync/secret-sync-recursive-fns.test.ts b/backend/src/services/secret-sync/secret-sync-recursive-fns.test.ts
index 4c86073213..f616061160 100644
--- a/backend/src/services/secret-sync/secret-sync-recursive-fns.test.ts
+++ b/backend/src/services/secret-sync/secret-sync-recursive-fns.test.ts
@@ -2,8 +2,8 @@ import {
assertWithinSecretLimit,
buildSyncPayload,
getAncestorPaths,
+ getSyncedFolders,
mergeImportedSecrets,
- resolveSyncFolders,
SECRET_SYNC_MAX_SECRETS
} from "./secret-sync-recursive-fns";
@@ -15,11 +15,11 @@ const deps = {
]
},
projectEnvDAL: { findOne: async () => ({ id: "env-1", slug: "dev", projectId: "proj-1" }) }
-} as unknown as Pick<Parameters<typeof resolveSyncFolders>[0], "folderDAL" | "projectEnvDAL">;
+} as unknown as Pick<Parameters<typeof getSyncedFolders>[0], "folderDAL" | "projectEnvDAL">;
-describe("resolveSyncFolders", () => {
+describe("getSyncedFolders", () => {
test("returns only the source folder when recursive is off", async () => {
- const folders = await resolveSyncFolders({
+ const folders = await getSyncedFolders({
...deps,
projectId: "proj-1",
environment: "dev",
@@ -32,7 +32,7 @@ describe("resolveSyncFolders", () => {
});
test("returns the source folder and its descendants when recursive is on", async () => {
- const folders = await resolveSyncFolders({
+ const folders = await getSyncedFolders({
...deps,
projectId: "proj-1",
environment: "dev",
@@ -147,7 +147,16 @@ describe("mergeImportedSecrets", () => {
expect(merged.map((entry) => entry.path).sort()).toEqual(["/backend", "/backend/api"]);
});
- test("a later import wins over an earlier one in the same folder, as today", () => {
+ // Two import statements in the same folder resolving the same key is pre-existing, released
+ // behavior: the reverse loop this reuses already shipped in secret-v2-bridge-service.ts. This
+ // pins that behavior so mergeImportedSecrets doesn't change it, not because it's the right
+ // design. Ideally a same-folder import collision would be a conflict error, the same way a
+ // duplicate name across folders already is.
+ //
+ // "Later" means higher position: secretImportDAL.findByFolderIds orders imports by position
+ // ascending, and that order survives unchanged through fnSecretsV2FromImports, so the import
+ // listed lower in that folder's import list is what wins.
+ test("the import listed later (by position) wins over an earlier one in the same folder", () => {
const merged = mergeImportedSecrets(
[],
[
@@ -226,7 +235,6 @@ describe("buildSyncPayload", () => {
projectFolderGrantDAL: { find: async () => [] },
actorOrgId: "org-1",
orgDAL: { findOrgById: async () => ({ allowCrossProjectSecretSharing: false }) },
- licenseService: { getPlan: async () => ({ crossProjectSecretSharing: false }) },
kmsService: {
createCipherPairWithDataKey: async () => ({
decryptor: () => "",
@@ -236,7 +244,7 @@ describe("buildSyncPayload", () => {
}
} as unknown as Parameters<typeof buildSyncPayload>[0];
- const payload = await buildSyncPayload(buildDeps, {
+ const entries = await buildSyncPayload(buildDeps, {
projectId: "proj-1",
environment: "dev",
sourcePath: "/",
@@ -245,13 +253,19 @@ describe("buildSyncPayload", () => {
includeImports: true
});
- const paths = payload.secrets.map((entry) => entry.path).sort();
+ const paths = entries.map((entry) => entry.path).sort();
expect(paths).toEqual(["/", "/api"]);
});
});
-describe("buildSyncPayload dedupeForRemoval", () => {
+// buildSyncPayload never dedupes or rejects a cross-folder name collision itself: whether that's
+// a hard conflict (sync/create) or something to look past (remove, so a sync that has drifted
+// into this state stays deletable) is a decision only the caller can make, made by wrapping this
+// function's raw entries with createSecretSyncPayload, optionally deduping them first. See
+// secret-sync-payload.test.ts for that composition, and secret-sync-queue.ts's
+// $getInfisicalSecrets for the one caller that dedupes.
+describe("buildSyncPayload cross-folder duplicates", () => {
const duplicateNameSecret = (folderId: string, value: string) => ({
id: `secret-${folderId}`,
key: "DB_URL",
@@ -284,7 +298,6 @@ describe("buildSyncPayload dedupeForRemoval", () => {
projectFolderGrantDAL: { find: async () => [] },
actorOrgId: "org-1",
orgDAL: { findOrgById: async () => ({ allowCrossProjectSecretSharing: false }) },
- licenseService: { getPlan: async () => ({ crossProjectSecretSharing: false }) },
kmsService: {
createCipherPairWithDataKey: async () => ({
decryptor: () => "",
@@ -303,20 +316,9 @@ describe("buildSyncPayload dedupeForRemoval", () => {
includeImports: true
};
- // Pins the sync-path requirement from the same bug: a name used in two folders must still
- // fail loudly on this path, since only the remove path may look past it.
- test("a cross-folder duplicate name still throws when dedupeForRemoval is not set", async () => {
- const payload = await buildSyncPayload(dedupeDeps, args);
+ test("returns both colliding entries as-is, neither deduped nor rejected", async () => {
+ const entries = await buildSyncPayload(dedupeDeps, args);
- expect(() => payload.flatten()).toThrow(/DB_URL/);
- });
-
- // A sync whose secrets collide across folders would otherwise be stuck: it fails to sync,
- // and until this test, it also failed to remove, which is the only way to delete it.
- test("dedupeForRemoval lets the remove path complete despite the same duplicate", async () => {
- const payload = await buildSyncPayload(dedupeDeps, { ...args, dedupeForRemoval: true });
-
- expect(() => payload.flatten()).not.toThrow();
- expect(Object.keys(payload.flatten())).toEqual(["DB_URL"]);
+ expect(entries.map((entry) => entry.path).sort()).toEqual(["/", "/api"]);
});
});
diff --git a/backend/src/services/secret-sync/secret-sync-recursive-fns.ts b/backend/src/services/secret-sync/secret-sync-recursive-fns.ts
index 1f1e173baf..9becc2f54a 100644
--- a/backend/src/services/secret-sync/secret-sync-recursive-fns.ts
+++ b/backend/src/services/secret-sync/secret-sync-recursive-fns.ts
@@ -7,12 +7,7 @@ import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-fold
import { TSecretImportDALFactory } from "@app/services/secret-import/secret-import-dal";
import { fnSecretsV2FromImports } from "@app/services/secret-import/secret-import-fns";
import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";
-import {
- createSecretSyncPayload,
- dedupeEntriesByDestinationKey,
- TSecretPayload,
- TSecretSyncPayload
-} from "@app/services/secret-sync/secret-sync-payload";
+import { TSecretPayload } from "@app/services/secret-sync/secret-sync-payload";
import { expandSecretReferencesFactory } from "@app/services/secret-v2-bridge/secret-reference-fns";
import { TSecretV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-dal";
import { recursivelyGetSecretPaths } from "@app/services/secret-v2-bridge/secret-v2-bridge-fns";
@@ -48,7 +43,11 @@ export const assertWithinSecretLimit = (count: number) => {
});
};
-export const resolveSyncFolders = async ({
+// A secret path is just a folder's location written as a filesystem-style string, so "folder"
+// and "path" name the same node here; each folder has exactly one path and each path names
+// exactly one folder, so this never returns the same folder twice. Non-recursive: just the
+// source folder. Recursive: the source folder plus every folder beneath it.
+export const getSyncedFolders = async ({
folderDAL,
projectEnvDAL,
projectId,
@@ -113,6 +112,13 @@ export const mergeImportedSecrets = (
return merged;
};
+// Resolves and decrypts every secret in the sync's source subtree (a single folder, or, when
+// recursive, that folder plus every folder beneath it), merging in any imports. This is the raw
+// ingredient list a TSecretSyncPayload is built from; it is deliberately not itself a
+// TSecretSyncPayload, since whether these entries should be deduped by destination key before
+// wrapping (the remove path needs that; every other caller wants a hard conflict instead) is a
+// decision specific to what each caller is about to do with them, not something this function
+// should know or care about.
export const buildSyncPayload = async (
deps: {
folderDAL: Pick<TSecretFolderDALFactory, "find" | "findByManySecretPath">;
@@ -129,19 +135,14 @@ export const buildSyncPayload = async (
sourcePath: string;
sourceFolderId: string;
recursive: boolean;
- keySchema?: string;
includeImports: boolean;
- // The remove path passes true so a duplicate name across folders cannot leave the sync
- // undeletable; every other caller must keep the duplicate-name check intact.
- dedupeForRemoval?: boolean;
}
-): Promise<TSecretSyncPayload> => {
+): Promise<TSecretPayload[]> => {
const { folderDAL, projectEnvDAL, secretV2BridgeDAL, secretImportDAL, expandSecretReferences, decryptSecretValue } =
deps;
- const { projectId, environment, sourcePath, sourceFolderId, recursive, keySchema, includeImports, dedupeForRemoval } =
- args;
+ const { projectId, environment, sourcePath, sourceFolderId, recursive, includeImports } = args;
- const folders = await resolveSyncFolders({
+ const folders = await getSyncedFolders({
folderDAL,
projectEnvDAL,
projectId,
@@ -194,10 +195,7 @@ export const buildSyncPayload = async (
);
if (!includeImports) {
- return createSecretSyncPayload(
- dedupeForRemoval ? dedupeEntriesByDestinationKey(entries, { environment, keySchema }) : entries,
- { environment, keySchema }
- );
+ return entries;
}
const secretImports = await secretImportDAL.findByFolderIds(folders.map(({ folderId }) => folderId));
@@ -242,8 +240,5 @@ export const buildSyncPayload = async (
assertWithinSecretLimit(allEntries.length);
}
- return createSecretSyncPayload(
- dedupeForRemoval ? dedupeEntriesByDestinationKey(allEntries, { environment, keySchema }) : allEntries,
- { environment, keySchema }
- );
+ return allEntries;
};
diff --git a/backend/src/services/secret-sync/secret-sync-schemas.test.ts b/backend/src/services/secret-sync/secret-sync-schemas.test.ts
index a8948f39f3..156b1bd55b 100644
--- a/backend/src/services/secret-sync/secret-sync-schemas.test.ts
+++ b/backend/src/services/secret-sync/secret-sync-schemas.test.ts
@@ -20,6 +20,9 @@ describe("BaseSyncOptionsSchema recursive", () => {
expect(result.success).toBe(true);
});
+ // Importing would need to write each secret back into the subfolder it came from, which isn't
+ // supported yet, so allowing this combination would squash every subfolder's secrets into the
+ // sync's root folder instead. See RECURSIVE_SYNC_REFINEMENT.
test("rejects recursive combined with importing from the destination", () => {
const result = schema.safeParse({
initialSyncBehavior: SecretSyncInitialSyncBehavior.ImportPrioritizeSource,
diff --git a/backend/src/services/secret-sync/secret-sync-schemas.ts b/backend/src/services/secret-sync/secret-sync-schemas.ts
index b2fbc0966b..7bbd35b5c0 100644
--- a/backend/src/services/secret-sync/secret-sync-schemas.ts
+++ b/backend/src/services/secret-sync/secret-sync-schemas.ts
@@ -9,6 +9,10 @@ import { SecretSync, SecretSyncInitialSyncBehavior } from "@app/services/secret-
import { SECRET_SYNC_CONNECTION_MAP, SECRET_SYNC_NAME_MAP } from "@app/services/secret-sync/secret-sync-maps";
import { TSyncOptionsConfig } from "@app/services/secret-sync/secret-sync-types";
+// Importing would need to write each secret back into the subfolder it came from, which we don't
+// support yet (hierarchical writes to destinations that can represent folders are planned for a
+// later PR). Until then, allowing this combination would silently squash every subfolder's
+// secrets into the sync's root folder, restructuring the user's existing secrets tree.
const RECURSIVE_SYNC_REFINEMENT = {
path: ["recursive"],
message:
@@ -76,15 +80,18 @@ const BaseSyncOptionsSchema = <T extends AnyZodObject | undefined = undefined>({
recursive: z.boolean().optional().describe(SecretSyncs.SYNC_OPTIONS(destination).recursive)
});
+ // What refinedSchema actually is: baseSchema, merged with the destination's own extra
+ // sync-option fields when `merge` supplies them, wrapped in a Zod effect that enforces
+ // isRecursiveCombinationAllowed across the result.
type TRefinedSchema = z.ZodEffects<
T extends AnyZodObject
? z.ZodObject<z.objectUtil.MergeShapes<typeof baseSchema.shape, T["shape"]>>
: typeof baseSchema
>;
- // `merge` is supplied exactly when `T` is a ZodObject, which is the condition this type branches
- // on, and the compiler cannot correlate a runtime ternary with a type parameter. Passing an
- // explicit type argument that disagrees with the `merge` argument would make the assertion false.
+ // The cast below is needed because TypeScript can't verify it on its own: every caller passes
+ // `merge` and a real T together or neither, so the runtime ternary and the type-level ternary
+ // above always agree, but the compiler has no way to prove that from a runtime value.
const refinedSchema = (
merge
? baseSchema.merge(merge).refine(isRecursiveCombinationAllowed, RECURSIVE_SYNC_REFINEMENT)
@@ -170,5 +177,10 @@ export const GenericUpdateSecretSyncFieldsSchema = <T extends AnyZodObject | und
.optional()
.describe(SecretSyncs.UPDATE(destination).secretPath),
isAutoSyncEnabled: z.boolean().optional().describe(SecretSyncs.UPDATE(destination).isAutoSyncEnabled),
+ // Optional like every other field here, so an update can touch just the name or description
+ // without resending sync options. This can't be used to sneak past
+ // isRecursiveCombinationAllowed: initialSyncBehavior inside BaseSyncOptionsSchema stays
+ // required, so a request that includes syncOptions at all must still submit a complete,
+ // valid one.
syncOptions: BaseSyncOptionsSchema({ destination, syncOptionsConfig, merge, isUpdateSchema: true }).optional()
});
diff --git a/backend/src/services/secret-sync/secret-sync-service.ts b/backend/src/services/secret-sync/secret-sync-service.ts
index 66e3830e05..5f0ec4e801 100644
--- a/backend/src/services/secret-sync/secret-sync-service.ts
+++ b/backend/src/services/secret-sync/secret-sync-service.ts
@@ -34,7 +34,8 @@ import {
preSaveTransformDestinationConfig,
preSaveTransformSyncOptions
} from "@app/services/secret-sync/secret-sync-fns";
-import { buildSyncPayload, resolveSyncFolders } from "@app/services/secret-sync/secret-sync-recursive-fns";
+import { createSecretSyncPayload } from "@app/services/secret-sync/secret-sync-payload";
+import { buildSyncPayload, getSyncedFolders } from "@app/services/secret-sync/secret-sync-recursive-fns";
import {
SecretSyncStatus,
TCheckDuplicateDestinationDTO,
@@ -162,7 +163,7 @@ export const secretSyncServiceFactory = ({
recursive: boolean;
}
) => {
- const folders = await resolveSyncFolders({
+ const folders = await getSyncedFolders({
folderDAL,
projectEnvDAL,
projectId,
@@ -247,7 +248,7 @@ export const secretSyncServiceFactory = ({
kmsService
});
- return buildSyncPayload(
+ const entries = await buildSyncPayload(
{
folderDAL,
projectEnvDAL,
@@ -268,11 +269,11 @@ export const secretSyncServiceFactory = ({
sourcePath,
sourceFolderId,
recursive: true,
- keySchema,
- includeImports: true,
- dedupeForRemoval: false
+ includeImports: true
}
);
+
+ return createSecretSyncPayload(entries, { environment, keySchema });
});
// Flattening a subtree onto a destination that holds one flat list can produce two secrets with