Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.
TypeScript acmeagentsai-agentscertificate-managementdevsecopsenvironment-variableskubernetesopen-sourcepampkipostgresprivate-caprivileged-access-managementsecret-managementsecret-managersecretssecrets-managementsecuritysecurity-toolsvault
avoid querying folder path on secret sync to reduce queries on folder table
adilsonc43ac7827e90
diff --git a/backend/src/services/secret-sync/secret-sync-dal.ts b/backend/src/services/secret-sync/secret-sync-dal.tsindex b2a2e63ad2..856a408e28 100644--- a/backend/src/services/secret-sync/secret-sync-dal.ts+++ b/backend/src/services/secret-sync/secret-sync-dal.ts@@ -11,6 +11,10 @@ export type TSecretSyncDALFactory = ReturnType<typeof secretSyncDALFactory>;type SecretSyncFindFilter = Parameters<typeof buildFindFilter<TSecretSyncs>>[0];+// Resolving a folder path loads every folder in the sync's environment. Callers that don't use the+// path, or already have it, pass false; the result then has `folder: null` even when the folder exists.+type TFolderPathOptions = { includeFolderPath?: boolean };+const baseSecretSyncQuery = ({ filter, db, tx }: { db: TDbClient; filter?: SecretSyncFindFilter; tx?: Knex }) => {const query = (tx || db.replicaNode())(TableName.SecretSync).leftJoin(TableName.SecretFolder, `${TableName.SecretSync}.folderId`, `${TableName.SecretFolder}.id`)@@ -118,7 +122,7 @@ export const secretSyncDALFactory = () => {const secretSyncOrm = ormify(db, TableName.SecretSync);- const findById = async (id: string, tx?: Knex) => {+ const findById = async (id: string, tx?: Knex, { includeFolderPath = true }: TFolderPathOptions = {}) => {try {const secretSync = await baseSecretSyncQuery({filter: { id },@@ -128,9 +132,10 @@ export const secretSyncDALFactory = (if (secretSync) {// TODO (scott): replace with cached folder path once implemented- const [folderWithPath] = secretSync.folderId- ? await folderDAL.findSecretPathByFolderIds(secretSync.projectId, [secretSync.folderId])- : [];+ const [folderWithPath] =+ includeFolderPath && secretSync.folderId+ ? await folderDAL.findSecretPathByFolderIds(secretSync.projectId, [secretSync.folderId])+ : [];return expandSecretSync(secretSync, folderWithPath);}} catch (error) {@@ -156,7 +161,11 @@ export const secretSyncDALFactory = (return expandSecretSync(secretSync, folderWithPath);};- const updateById = async (syncId: string, data: Parameters<(typeof secretSyncOrm)["updateById"]>[1]) => {+ const updateById = async (+ syncId: string,+ data: Parameters<(typeof secretSyncOrm)["updateById"]>[1],+ { includeFolderPath = true }: TFolderPathOptions = {}+ ) => {const secretSync = (await secretSyncOrm.transaction(async (tx) => {const sync = await secretSyncOrm.updateById(syncId, data, tx);@@ -168,9 +177,10 @@ export const secretSyncDALFactory = (}))!;// TODO (scott): replace with cached folder path once implemented- const [folderWithPath] = secretSync.folderId- ? await folderDAL.findSecretPathByFolderIds(secretSync.projectId, [secretSync.folderId])- : [];+ const [folderWithPath] =+ includeFolderPath && secretSync.folderId+ ? await folderDAL.findSecretPathByFolderIds(secretSync.projectId, [secretSync.folderId])+ : [];return expandSecretSync(secretSync, folderWithPath);};@@ -190,16 +200,22 @@ export const secretSyncDALFactory = (}};- const find = async (filter: Parameters<(typeof secretSyncOrm)["find"]>[0], tx?: Knex) => {+ const find = async (+ filter: Parameters<(typeof secretSyncOrm)["find"]>[0],+ tx?: Knex,+ { includeFolderPath = true }: TFolderPathOptions = {}+ ) => {try {const secretSyncs = await baseSecretSyncQuery({ filter, db, tx });if (!secretSyncs.length) return [];- const foldersWithPath = await folderDAL.findSecretPathByFolderIds(- secretSyncs[0].projectId,- secretSyncs.filter((sync) => Boolean(sync.folderId)).map((sync) => sync.folderId!)- );+ const foldersWithPath = includeFolderPath+ ? await folderDAL.findSecretPathByFolderIds(+ secretSyncs[0].projectId,+ secretSyncs.filter((sync) => Boolean(sync.folderId)).map((sync) => sync.folderId!)+ )+ : [];// TODO (scott): replace with cached folder path once implementedconst folderRecord: Record<string, (typeof foldersWithPath)[number]> = {};diff --git a/backend/src/services/secret-sync/secret-sync-queue.ts b/backend/src/services/secret-sync/secret-sync-queue.tsindex a09cca8754..44f8af94a8 100644--- a/backend/src/services/secret-sync/secret-sync-queue.ts+++ b/backend/src/services/secret-sync/secret-sync-queue.ts@@ -505,9 +505,13 @@ export const secretSyncQueueFactory = ({"Failed to sync secrets due to plan restriction. Upgrade plan to access enterprise secret syncs.");- await secretSyncDAL.updateById(syncId, {- syncStatus: SecretSyncStatus.Running- });+ await secretSyncDAL.updateById(+ syncId,+ {+ syncStatus: SecretSyncStatus.Running+ },+ { includeFolderPath: false }+ );logger.info(`SecretSync Sync [syncId=${secretSync.id}] [destination=${secretSync.destination}] [projectId=${secretSync.projectId}] [folderId=${secretSync.folderId}] [connectionId=${secretSync.connectionId}]`@@ -637,12 +641,16 @@ export const secretSyncQueueFactory = ({});if (isSynced || isFinalAttempt) {- const updatedSecretSync = await secretSyncDAL.updateById(secretSync.id, {- syncStatus,- lastSyncJobId: job.id,- lastSyncMessage: syncMessage,- lastSyncedAt: isSynced ? ranAt : undefined- });+ const updatedSecretSync = await secretSyncDAL.updateById(+ secretSync.id,+ {+ syncStatus,+ lastSyncJobId: job.id,+ lastSyncMessage: syncMessage,+ lastSyncedAt: isSynced ? ranAt : undefined+ },+ { includeFolderPath: false }+ );if (!isSynced) {void telemetryService@@ -659,7 +667,7 @@ export const secretSyncQueueFactory = ({.catch(() => {});await $queueSendSecretSyncFailedNotifications({- secretSync: updatedSecretSync,+ secretSync: { ...updatedSecretSync, folder: secretSync.folder },action: SecretSyncAction.SyncSecrets,auditLogInfo});@@ -675,9 +683,13 @@ export const secretSyncQueueFactory = ({data: { syncId, auditLogInfo, importBehavior }} = job;- await secretSyncDAL.updateById(syncId, {- importStatus: SecretSyncStatus.Running- });+ await secretSyncDAL.updateById(+ syncId,+ {+ importStatus: SecretSyncStatus.Running+ },+ { includeFolderPath: false }+ );logger.info(`SecretSync Import [syncId=${secretSync.id}] [destination=${secretSync.destination}] [projectId=${secretSync.projectId}] [folderId=${secretSync.folderId}] [connectionId=${secretSync.connectionId}]`@@ -771,16 +783,20 @@ export const secretSyncQueueFactory = ({});if (isSuccess || isFinalAttempt) {- const updatedSecretSync = await secretSyncDAL.updateById(secretSync.id, {- importStatus,- lastImportJobId: job.id,- lastImportMessage: importMessage,- lastImportedAt: isSuccess ? ranAt : undefined- });+ const updatedSecretSync = await secretSyncDAL.updateById(+ secretSync.id,+ {+ importStatus,+ lastImportJobId: job.id,+ lastImportMessage: importMessage,+ lastImportedAt: isSuccess ? ranAt : undefined+ },+ { includeFolderPath: false }+ );if (!isSuccess) {await $queueSendSecretSyncFailedNotifications({- secretSync: updatedSecretSync,+ secretSync: { ...updatedSecretSync, folder: secretSync.folder },action: SecretSyncAction.ImportSecrets,auditLogInfo});@@ -803,9 +819,13 @@ export const secretSyncQueueFactory = ({"Failed to remove secrets due to plan restriction. Upgrade plan to access enterprise secret syncs.");- await secretSyncDAL.updateById(syncId, {- removeStatus: SecretSyncStatus.Running- });+ await secretSyncDAL.updateById(+ syncId,+ {+ removeStatus: SecretSyncStatus.Running+ },+ { includeFolderPath: false }+ );logger.info(`SecretSync Remove [syncId=${secretSync.id}] [destination=${secretSync.destination}] [projectId=${secretSync.projectId}] [folderId=${secretSync.folderId}] [connectionId=${secretSync.connectionId}]`@@ -912,16 +932,20 @@ export const secretSyncQueueFactory = ({if (isSuccess && deleteSyncOnComplete) {await secretSyncDAL.deleteById(secretSync.id);} else {- const updatedSecretSync = await secretSyncDAL.updateById(secretSync.id, {- removeStatus,- lastRemoveJobId: job.id,- lastRemoveMessage: removeMessage,- lastRemovedAt: isSuccess ? ranAt : undefined- });+ const updatedSecretSync = await secretSyncDAL.updateById(+ secretSync.id,+ {+ removeStatus,+ lastRemoveJobId: job.id,+ lastRemoveMessage: removeMessage,+ lastRemovedAt: isSuccess ? ranAt : undefined+ },+ { includeFolderPath: false }+ );if (!isSuccess) {await $queueSendSecretSyncFailedNotifications({- secretSync: updatedSecretSync,+ secretSync: { ...updatedSecretSync, folder: secretSync.folder },action: SecretSyncAction.RemoveSecrets,auditLogInfo});@@ -1060,10 +1084,14 @@ export const secretSyncQueueFactory = ({)).filter((entry): entry is NonNullable<typeof entry> => Boolean(entry));- const candidateSyncs = await secretSyncDAL.find({- $in: { folderId: [...ancestorFolders.map((entry) => entry.id), ...(folder ? [folder.id] : [])] },- isAutoSyncEnabled: true- });+ const candidateSyncs = await secretSyncDAL.find(+ {+ $in: { folderId: [...ancestorFolders.map((entry) => entry.id), ...(folder ? [folder.id] : [])] },+ isAutoSyncEnabled: true+ },+ undefined,+ { includeFolderPath: false }+ );// A sync on the path itself always matches, whether or not it includes subfolders. A sync on an// ancestor folder only matches when it includes them, so a sync rooted above this path that does@@ -1219,15 +1247,15 @@ export const secretSyncQueueFactory = ({| TQueueSecretSyncImportSecretsByIdDTO| TQueueSecretSyncRemoveSecretsByIdDTO;- const secretSync = await secretSyncDAL.findById(syncId);+ const secretSyncWithoutPath = await secretSyncDAL.findById(syncId, undefined, { includeFolderPath: false });- if (!secretSync) {+ if (!secretSyncWithoutPath) {// skip rather than throw, so it doesn't retry-storm when a sync is deleted.- logger.info(`AppConnectionSecretSync: secret sync ${syncId} not found (deleted?), skipping`);+ logger.info(`AppConnectionSecretSync: secret sync ${syncId} not found, skipping`);return;}- const { connectionId } = secretSync;+ const { connectionId } = secretSyncWithoutPath;let lock: Awaited<ReturnType<typeof keyStore.acquireLock>>;@@ -1263,6 +1291,13 @@ export const secretSyncQueueFactory = ({}}+ const secretSync = await secretSyncDAL.findById(syncId);++ if (!secretSync) {+ logger.info(`AppConnectionSecretSync: secret sync ${syncId} not found, skipping`);+ return;+ }+switch (job.name) {case QueueJobs.SecretSyncSyncSecrets: {await $handleSyncSecretsJob(job as TSecretSyncSyncSecretsDTO, secretSync);