Skip to content

Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.

avoid querying folder path on secret sync to reduce queries on folder table

adilsonc43ac7827e90
diff --git a/backend/src/services/secret-sync/secret-sync-dal.ts b/backend/src/services/secret-sync/secret-sync-dal.ts
index b2a2e63ad2..856a408e28 100644
--- a/backend/src/services/secret-sync/secret-sync-dal.ts
+++ b/backend/src/services/secret-sync/secret-sync-dal.ts
@@ -11,6 +11,10 @@ export type TSecretSyncDALFactory = ReturnType<typeof secretSyncDALFactory>;
type SecretSyncFindFilter = Parameters<typeof buildFindFilter<TSecretSyncs>>[0];
+// Resolving a folder path loads every folder in the sync's environment. Callers that don't use the
+// path, or already have it, pass false; the result then has `folder: null` even when the folder exists.
+type TFolderPathOptions = { includeFolderPath?: boolean };
+
const baseSecretSyncQuery = ({ filter, db, tx }: { db: TDbClient; filter?: SecretSyncFindFilter; tx?: Knex }) => {
const query = (tx || db.replicaNode())(TableName.SecretSync)
.leftJoin(TableName.SecretFolder, `${TableName.SecretSync}.folderId`, `${TableName.SecretFolder}.id`)
@@ -118,7 +122,7 @@ export const secretSyncDALFactory = (
) => {
const secretSyncOrm = ormify(db, TableName.SecretSync);
- const findById = async (id: string, tx?: Knex) => {
+ const findById = async (id: string, tx?: Knex, { includeFolderPath = true }: TFolderPathOptions = {}) => {
try {
const secretSync = await baseSecretSyncQuery({
filter: { id },
@@ -128,9 +132,10 @@ export const secretSyncDALFactory = (
if (secretSync) {
// TODO (scott): replace with cached folder path once implemented
- const [folderWithPath] = secretSync.folderId
- ? await folderDAL.findSecretPathByFolderIds(secretSync.projectId, [secretSync.folderId])
- : [];
+ const [folderWithPath] =
+ includeFolderPath && secretSync.folderId
+ ? await folderDAL.findSecretPathByFolderIds(secretSync.projectId, [secretSync.folderId])
+ : [];
return expandSecretSync(secretSync, folderWithPath);
}
} catch (error) {
@@ -156,7 +161,11 @@ export const secretSyncDALFactory = (
return expandSecretSync(secretSync, folderWithPath);
};
- const updateById = async (syncId: string, data: Parameters<(typeof secretSyncOrm)["updateById"]>[1]) => {
+ const updateById = async (
+ syncId: string,
+ data: Parameters<(typeof secretSyncOrm)["updateById"]>[1],
+ { includeFolderPath = true }: TFolderPathOptions = {}
+ ) => {
const secretSync = (await secretSyncOrm.transaction(async (tx) => {
const sync = await secretSyncOrm.updateById(syncId, data, tx);
@@ -168,9 +177,10 @@ export const secretSyncDALFactory = (
}))!;
// TODO (scott): replace with cached folder path once implemented
- const [folderWithPath] = secretSync.folderId
- ? await folderDAL.findSecretPathByFolderIds(secretSync.projectId, [secretSync.folderId])
- : [];
+ const [folderWithPath] =
+ includeFolderPath && secretSync.folderId
+ ? await folderDAL.findSecretPathByFolderIds(secretSync.projectId, [secretSync.folderId])
+ : [];
return expandSecretSync(secretSync, folderWithPath);
};
@@ -190,16 +200,22 @@ export const secretSyncDALFactory = (
}
};
- const find = async (filter: Parameters<(typeof secretSyncOrm)["find"]>[0], tx?: Knex) => {
+ const find = async (
+ filter: Parameters<(typeof secretSyncOrm)["find"]>[0],
+ tx?: Knex,
+ { includeFolderPath = true }: TFolderPathOptions = {}
+ ) => {
try {
const secretSyncs = await baseSecretSyncQuery({ filter, db, tx });
if (!secretSyncs.length) return [];
- const foldersWithPath = await folderDAL.findSecretPathByFolderIds(
- secretSyncs[0].projectId,
- secretSyncs.filter((sync) => Boolean(sync.folderId)).map((sync) => sync.folderId!)
- );
+ const foldersWithPath = includeFolderPath
+ ? await folderDAL.findSecretPathByFolderIds(
+ secretSyncs[0].projectId,
+ secretSyncs.filter((sync) => Boolean(sync.folderId)).map((sync) => sync.folderId!)
+ )
+ : [];
// TODO (scott): replace with cached folder path once implemented
const folderRecord: Record<string, (typeof foldersWithPath)[number]> = {};
diff --git a/backend/src/services/secret-sync/secret-sync-queue.ts b/backend/src/services/secret-sync/secret-sync-queue.ts
index a09cca8754..44f8af94a8 100644
--- a/backend/src/services/secret-sync/secret-sync-queue.ts
+++ b/backend/src/services/secret-sync/secret-sync-queue.ts
@@ -505,9 +505,13 @@ export const secretSyncQueueFactory = ({
"Failed to sync secrets due to plan restriction. Upgrade plan to access enterprise secret syncs."
);
- await secretSyncDAL.updateById(syncId, {
- syncStatus: SecretSyncStatus.Running
- });
+ await secretSyncDAL.updateById(
+ syncId,
+ {
+ syncStatus: SecretSyncStatus.Running
+ },
+ { includeFolderPath: false }
+ );
logger.info(
`SecretSync Sync [syncId=${secretSync.id}] [destination=${secretSync.destination}] [projectId=${secretSync.projectId}] [folderId=${secretSync.folderId}] [connectionId=${secretSync.connectionId}]`
@@ -637,12 +641,16 @@ export const secretSyncQueueFactory = ({
});
if (isSynced || isFinalAttempt) {
- const updatedSecretSync = await secretSyncDAL.updateById(secretSync.id, {
- syncStatus,
- lastSyncJobId: job.id,
- lastSyncMessage: syncMessage,
- lastSyncedAt: isSynced ? ranAt : undefined
- });
+ const updatedSecretSync = await secretSyncDAL.updateById(
+ secretSync.id,
+ {
+ syncStatus,
+ lastSyncJobId: job.id,
+ lastSyncMessage: syncMessage,
+ lastSyncedAt: isSynced ? ranAt : undefined
+ },
+ { includeFolderPath: false }
+ );
if (!isSynced) {
void telemetryService
@@ -659,7 +667,7 @@ export const secretSyncQueueFactory = ({
.catch(() => {});
await $queueSendSecretSyncFailedNotifications({
- secretSync: updatedSecretSync,
+ secretSync: { ...updatedSecretSync, folder: secretSync.folder },
action: SecretSyncAction.SyncSecrets,
auditLogInfo
});
@@ -675,9 +683,13 @@ export const secretSyncQueueFactory = ({
data: { syncId, auditLogInfo, importBehavior }
} = job;
- await secretSyncDAL.updateById(syncId, {
- importStatus: SecretSyncStatus.Running
- });
+ await secretSyncDAL.updateById(
+ syncId,
+ {
+ importStatus: SecretSyncStatus.Running
+ },
+ { includeFolderPath: false }
+ );
logger.info(
`SecretSync Import [syncId=${secretSync.id}] [destination=${secretSync.destination}] [projectId=${secretSync.projectId}] [folderId=${secretSync.folderId}] [connectionId=${secretSync.connectionId}]`
@@ -771,16 +783,20 @@ export const secretSyncQueueFactory = ({
});
if (isSuccess || isFinalAttempt) {
- const updatedSecretSync = await secretSyncDAL.updateById(secretSync.id, {
- importStatus,
- lastImportJobId: job.id,
- lastImportMessage: importMessage,
- lastImportedAt: isSuccess ? ranAt : undefined
- });
+ const updatedSecretSync = await secretSyncDAL.updateById(
+ secretSync.id,
+ {
+ importStatus,
+ lastImportJobId: job.id,
+ lastImportMessage: importMessage,
+ lastImportedAt: isSuccess ? ranAt : undefined
+ },
+ { includeFolderPath: false }
+ );
if (!isSuccess) {
await $queueSendSecretSyncFailedNotifications({
- secretSync: updatedSecretSync,
+ secretSync: { ...updatedSecretSync, folder: secretSync.folder },
action: SecretSyncAction.ImportSecrets,
auditLogInfo
});
@@ -803,9 +819,13 @@ export const secretSyncQueueFactory = ({
"Failed to remove secrets due to plan restriction. Upgrade plan to access enterprise secret syncs."
);
- await secretSyncDAL.updateById(syncId, {
- removeStatus: SecretSyncStatus.Running
- });
+ await secretSyncDAL.updateById(
+ syncId,
+ {
+ removeStatus: SecretSyncStatus.Running
+ },
+ { includeFolderPath: false }
+ );
logger.info(
`SecretSync Remove [syncId=${secretSync.id}] [destination=${secretSync.destination}] [projectId=${secretSync.projectId}] [folderId=${secretSync.folderId}] [connectionId=${secretSync.connectionId}]`
@@ -912,16 +932,20 @@ export const secretSyncQueueFactory = ({
if (isSuccess && deleteSyncOnComplete) {
await secretSyncDAL.deleteById(secretSync.id);
} else {
- const updatedSecretSync = await secretSyncDAL.updateById(secretSync.id, {
- removeStatus,
- lastRemoveJobId: job.id,
- lastRemoveMessage: removeMessage,
- lastRemovedAt: isSuccess ? ranAt : undefined
- });
+ const updatedSecretSync = await secretSyncDAL.updateById(
+ secretSync.id,
+ {
+ removeStatus,
+ lastRemoveJobId: job.id,
+ lastRemoveMessage: removeMessage,
+ lastRemovedAt: isSuccess ? ranAt : undefined
+ },
+ { includeFolderPath: false }
+ );
if (!isSuccess) {
await $queueSendSecretSyncFailedNotifications({
- secretSync: updatedSecretSync,
+ secretSync: { ...updatedSecretSync, folder: secretSync.folder },
action: SecretSyncAction.RemoveSecrets,
auditLogInfo
});
@@ -1060,10 +1084,14 @@ export const secretSyncQueueFactory = ({
)
).filter((entry): entry is NonNullable<typeof entry> => Boolean(entry));
- const candidateSyncs = await secretSyncDAL.find({
- $in: { folderId: [...ancestorFolders.map((entry) => entry.id), ...(folder ? [folder.id] : [])] },
- isAutoSyncEnabled: true
- });
+ const candidateSyncs = await secretSyncDAL.find(
+ {
+ $in: { folderId: [...ancestorFolders.map((entry) => entry.id), ...(folder ? [folder.id] : [])] },
+ isAutoSyncEnabled: true
+ },
+ undefined,
+ { includeFolderPath: false }
+ );
// A sync on the path itself always matches, whether or not it includes subfolders. A sync on an
// ancestor folder only matches when it includes them, so a sync rooted above this path that does
@@ -1219,15 +1247,15 @@ export const secretSyncQueueFactory = ({
| TQueueSecretSyncImportSecretsByIdDTO
| TQueueSecretSyncRemoveSecretsByIdDTO;
- const secretSync = await secretSyncDAL.findById(syncId);
+ const secretSyncWithoutPath = await secretSyncDAL.findById(syncId, undefined, { includeFolderPath: false });
- if (!secretSync) {
+ if (!secretSyncWithoutPath) {
// skip rather than throw, so it doesn't retry-storm when a sync is deleted.
- logger.info(`AppConnectionSecretSync: secret sync ${syncId} not found (deleted?), skipping`);
+ logger.info(`AppConnectionSecretSync: secret sync ${syncId} not found, skipping`);
return;
}
- const { connectionId } = secretSync;
+ const { connectionId } = secretSyncWithoutPath;
let lock: Awaited<ReturnType<typeof keyStore.acquireLock>>;
@@ -1263,6 +1291,13 @@ export const secretSyncQueueFactory = ({
}
}
+ const secretSync = await secretSyncDAL.findById(syncId);
+
+ if (!secretSync) {
+ logger.info(`AppConnectionSecretSync: secret sync ${syncId} not found, skipping`);
+ return;
+ }
+
switch (job.name) {
case QueueJobs.SecretSyncSyncSecrets: {
await $handleSyncSecretsJob(job as TSecretSyncSyncSecretsDTO, secretSync);