Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.
TypeScript acmeagentsai-agentscertificate-managementdevsecopsenvironment-variableskubernetesopen-sourcepampkipostgresprivate-caprivileged-access-managementsecret-managementsecret-managersecretssecrets-managementsecuritysecurity-toolsvault
import { AxiosError, isAxiosError } from "axios";
import { Job } from "bullmq";
import { randomUUID } from "crypto";
import { ProjectMembershipRole, SecretType } from "@app/db/schemas";
import { EventType, TAuditLogServiceFactory } from "@app/ee/services/audit-log/audit-log-types";
import { TGatewayPoolServiceFactory } from "@app/ee/services/gateway-pool/gateway-pool-service";
import { TGatewayV2ServiceFactory } from "@app/ee/services/gateway-v2/gateway-v2-service";
import { TLicenseServiceFactory } from "@app/ee/services/license/license-service";
import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";
import { getConfig } from "@app/lib/config/env";
import { CronJobName, TCronJobFactory } from "@app/lib/cron/cron-job";
import { logger } from "@app/lib/logger";
import {
highCardinalityMeter,
recordSecretSyncOutcomeMetric,
recordSecretSyncStatusChangeMetric
} from "@app/lib/telemetry/metrics";
import { triggerWorkflowIntegrationNotification } from "@app/lib/workflow-integrations/trigger-notification";
import { TriggerFeature } from "@app/lib/workflow-integrations/types";
import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
import { SecretNameSchema } from "@app/server/lib/schemas";
import { decryptAppConnectionCredentials } from "@app/services/app-connection/app-connection-fns";
import { ActorType } from "@app/services/auth/auth-type";
import { TEventEmitter } from "@app/services/event-outbox/event-outbox-types";
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
import { KmsDataKey } from "@app/services/kms/kms-types";
import { TOidcIssuerServiceFactory } from "@app/services/oidc-issuer/oidc-issuer-service";
import { TOrgDALFactory } from "@app/services/org/org-dal";
import { TProjectBotDALFactory } from "@app/services/project-bot/project-bot-dal";
import { TProjectEnvDALFactory } from "@app/services/project-env/project-env-dal";
import { TProjectFolderGrantDALFactory } from "@app/services/project-folder-grant/project-folder-grant-dal";
import { TProjectMembershipDALFactory } from "@app/services/project-membership/project-membership-dal";
import { TProjectDALFactory } from "@app/services/project/project-dal";
import { TResourceMetadataDALFactory } from "@app/services/resource-metadata/resource-metadata-dal";
import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
import { TSecretImportDALFactory } from "@app/services/secret-import/secret-import-dal";
import { resolveAzureKeyVaultImportedSecretKey } from "@app/services/secret-sync/azure-key-vault/azure-key-vault-secret-name";
import { TSecretSyncDALFactory } from "@app/services/secret-sync/secret-sync-dal";
import {
SecretSync,
SecretSyncImportBehavior,
SecretSyncInitialSyncBehavior
} from "@app/services/secret-sync/secret-sync-enums";
import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";
import { enterpriseSyncCheck, parseSyncErrorMessage, SecretSyncFns } from "@app/services/secret-sync/secret-sync-fns";
import { SECRET_SYNC_DAILY_RETRY_DESTINATIONS, SECRET_SYNC_NAME_MAP } from "@app/services/secret-sync/secret-sync-maps";
import { buildSyncPayload, getAncestorPaths } from "@app/services/secret-sync/secret-sync-recursive-fns";
import {
SecretSyncAction,
SecretSyncStatus,
TQueueSecretSyncImportSecretsByIdDTO,
TQueueSecretSyncRemoveSecretsByIdDTO,
TQueueSecretSyncsByPathDTO,
TQueueSecretSyncSyncSecretsByIdDTO,
TQueueSendSecretSyncActionFailedNotificationsDTO,
TSecretMap,
TSecretSync,
TSecretSyncImportSecretsDTO,
TSecretSyncRaw,
TSecretSyncRemoveSecretsDTO,
TSecretSyncSyncSecretsDTO,
TSecretSyncWithCredentials,
TSendSecretSyncFailedNotificationsJobDTO,
TSyncSecretsResult
} from "@app/services/secret-sync/secret-sync-types";
import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
import { expandSecretReferencesFactory } from "@app/services/secret-v2-bridge/secret-reference-fns";
import { TSecretV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-dal";
import { TSecretVersionV2DALFactory } from "@app/services/secret-v2-bridge/secret-version-dal";
import { TSecretVersionV2TagDALFactory } from "@app/services/secret-v2-bridge/secret-version-tag-dal";
import { TSecretDALFactory } from "@app/services/secret/secret-dal";
import { createManySecretsRawFnFactory, updateManySecretsRawFnFactory } from "@app/services/secret/secret-fns";
import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service";
import { TTelemetryServiceFactory } from "@app/services/telemetry/telemetry-service";
import { PostHogEventTypes } from "@app/services/telemetry/telemetry-types";
import { TAppConnectionDALFactory } from "../app-connection/app-connection-dal";
import { TFolderCommitServiceFactory } from "../folder-commit/folder-commit-service";
import { TGitHubAppDALFactory } from "../github-app/github-app-dal";
import { TMicrosoftTeamsServiceFactory } from "../microsoft-teams/microsoft-teams-service";
import { TProjectMicrosoftTeamsConfigDALFactory } from "../microsoft-teams/project-microsoft-teams-config-dal";
import { TNotificationServiceFactory } from "../notification/notification-service";
import { NotificationType } from "../notification/notification-types";
import { TProjectSlackConfigDALFactory } from "../slack/project-slack-config-dal";
const DEFAULT_SECRET_SYNC_RETRY_CONFIG = {
attempts: 5,
backoff: { type: "exponential" as const, delay: 3000 }
};
export type TSecretSyncQueueFactory = ReturnType<typeof secretSyncQueueFactory>;
type TSecretSyncQueueFactoryDep = {
queueService: Pick<TQueueServiceFactory, "queue" | "start">;
cronJob: TCronJobFactory;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
appConnectionDAL: Pick<TAppConnectionDALFactory, "findById" | "update" | "updateById">;
keyStore: Pick<TKeyStoreFactory, "acquireLock" | "incrementByAndRefreshExpiryIfUnderLimit" | "decrementByOrDelete">;
gitHubAppDAL: Pick<TGitHubAppDALFactory, "findOne">;
folderDAL: TSecretFolderDALFactory;
projectEnvDAL: Pick<TProjectEnvDALFactory, "findOne">;
secretV2BridgeDAL: Pick<
TSecretV2BridgeDALFactory,
| "findByFolderId"
| "findByFolderIds"
| "find"
| "insertMany"
| "upsertSecretReferences"
| "findBySecretKeys"
| "bulkUpdate"
| "deleteMany"
| "invalidateSecretCacheByProjectId"
>;
secretImportDAL: Pick<TSecretImportDALFactory, "findByFolderIds" | "findByIds">;
secretSyncDAL: Pick<
TSecretSyncDALFactory,
"findById" | "find" | "findConnectionId" | "updateById" | "deleteById" | "update" | "updateAndReturnIds"
>;
auditLogService: Pick<TAuditLogServiceFactory, "createAuditLog">;
projectMembershipDAL: Pick<TProjectMembershipDALFactory, "findAllProjectMembers">;
projectDAL: TProjectDALFactory;
smtpService: Pick<TSmtpService, "sendMail">;
projectBotDAL: TProjectBotDALFactory;
secretDAL: TSecretDALFactory;
secretVersionDAL: TSecretVersionDALFactory;
secretBlindIndexDAL: TSecretBlindIndexDALFactory;
secretTagDAL: TSecretTagDALFactory;
secretVersionTagDAL: TSecretVersionTagDALFactory;
secretVersionV2BridgeDAL: Pick<TSecretVersionV2DALFactory, "insertMany" | "findLatestVersionMany">;
secretVersionTagV2BridgeDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany">;
resourceMetadataDAL: Pick<TResourceMetadataDALFactory, "insertMany" | "delete">;
folderCommitService: Pick<TFolderCommitServiceFactory, "createCommit">;
eventEmitter: TEventEmitter;
licenseService: Pick<TLicenseServiceFactory, "getPlan">;
gatewayV2Service: Pick<TGatewayV2ServiceFactory, "getPlatformConnectionDetailsByGatewayId">;
gatewayPoolService: Pick<TGatewayPoolServiceFactory, "resolveEffectiveGatewayId">;
notificationService: Pick<TNotificationServiceFactory, "createUserNotifications">;
projectSlackConfigDAL: Pick<TProjectSlackConfigDALFactory, "getIntegrationDetailsByProject">;
projectMicrosoftTeamsConfigDAL: Pick<TProjectMicrosoftTeamsConfigDALFactory, "getIntegrationDetailsByProject">;
microsoftTeamsService: Pick<TMicrosoftTeamsServiceFactory, "sendNotification">;
telemetryService: Pick<TTelemetryServiceFactory, "sendPostHogEvents">;
projectFolderGrantDAL: Pick<TProjectFolderGrantDALFactory, "find">;
orgDAL: Pick<TOrgDALFactory, "findOrgById">;
oidcIssuerService: Pick<TOidcIssuerServiceFactory, "signToken">;
};
type SecretSyncActionJob = Job<
TQueueSecretSyncSyncSecretsByIdDTO | TQueueSecretSyncImportSecretsByIdDTO | TQueueSecretSyncRemoveSecretsByIdDTO
>;
const JITTER_MS = 10 * 1000;
const REQUEUE_MS = 30 * 1000;
const REQUEUE_LIMIT = 120;
const CONNECTION_CONCURRENCY_LIMIT = 5;
const CONNECTION_CONCURRENCY_TTL_SECONDS = (REQUEUE_MS * REQUEUE_LIMIT) / 1000 / 2;
const getRequeueDelay = (failureCount?: number) => {
const jitter = Math.random() * JITTER_MS;
if (!failureCount) return jitter;
return REQUEUE_MS + jitter;
};
export const secretSyncQueueFactory = ({
queueService,
cronJob,
kmsService,
appConnectionDAL,
gitHubAppDAL,
keyStore,
folderDAL,
projectEnvDAL,
secretV2BridgeDAL,
secretImportDAL,
secretSyncDAL,
auditLogService,
projectMembershipDAL,
projectDAL,
smtpService,
projectBotDAL,
secretDAL,
secretVersionDAL,
secretBlindIndexDAL,
secretTagDAL,
secretVersionTagDAL,
secretVersionV2BridgeDAL,
secretVersionTagV2BridgeDAL,
resourceMetadataDAL,
folderCommitService,
eventEmitter,
licenseService,
gatewayV2Service,
gatewayPoolService,
notificationService,
projectSlackConfigDAL,
projectMicrosoftTeamsConfigDAL,
microsoftTeamsService,
telemetryService,
projectFolderGrantDAL,
orgDAL,
oidcIssuerService
}: TSecretSyncQueueFactoryDep) => {
const appCfg = getConfig();
const integrationMeter = highCardinalityMeter("SecretSyncs");
const syncSecretsErrorHistogram = integrationMeter.createHistogram("secret_sync_sync_secrets_errors", {
description: "Secret Sync - sync secrets errors",
unit: "1"
});
const importSecretsErrorHistogram = integrationMeter.createHistogram("secret_sync_import_secrets_errors", {
description: "Secret Sync - import secrets errors",
unit: "1"
});
const removeSecretsErrorHistogram = integrationMeter.createHistogram("secret_sync_remove_secrets_errors", {
description: "Secret Sync - remove secrets errors",
unit: "1"
});
const $createManySecretsRawFn = createManySecretsRawFnFactory({
projectDAL,
projectBotDAL,
secretDAL,
secretVersionDAL,
secretBlindIndexDAL,
secretTagDAL,
secretVersionTagDAL,
folderDAL,
kmsService,
secretVersionV2BridgeDAL,
secretV2BridgeDAL,
secretVersionTagV2BridgeDAL,
resourceMetadataDAL,
folderCommitService,
eventEmitter
});
const $updateManySecretsRawFn = updateManySecretsRawFnFactory({
projectDAL,
projectBotDAL,
secretDAL,
secretVersionDAL,
secretBlindIndexDAL,
secretTagDAL,
secretVersionTagDAL,
folderDAL,
kmsService,
secretVersionV2BridgeDAL,
secretV2BridgeDAL,
secretVersionTagV2BridgeDAL,
resourceMetadataDAL,
folderCommitService,
eventEmitter
});
// INCR + cap check + EXPIRE happen atomically inside a Lua script. TTL is refreshed
// only when a slot is actually claimed; failed probes (over-limit) and releases
// never touch EXPIRE — so an orphaned counter from a worker that died mid-sync
// can't have its expiry indefinitely shoved forward by subsequent admit attempts,
// and ages out within CONNECTION_CONCURRENCY_TTL_SECONDS of the last successful
// admit.
const $tryAdmitConnectionConcurrency = async (connectionId: string) => {
const key = KeyStorePrefixes.AppConnectionConcurrentJobs(connectionId);
const count = await keyStore.incrementByAndRefreshExpiryIfUnderLimit(
key,
CONNECTION_CONCURRENCY_LIMIT,
CONNECTION_CONCURRENCY_TTL_SECONDS
);
return count !== -1;
};
const $releaseConnectionConcurrency = async (connectionId: string) => {
const key = KeyStorePrefixes.AppConnectionConcurrentJobs(connectionId);
await keyStore.decrementByOrDelete(key);
};
const $getInfisicalSecrets = async (
secretSync: TSecretSyncRaw | TSecretSyncWithCredentials,
includeImports = true
) => {
const { projectId, folderId, environment, folder } = secretSync;
if (!folderId || !environment || !folder)
throw new SecretSyncError({
message:
"Invalid Secret Sync source configuration: folder no longer exists. Please update source environment and secret path.",
shouldRetry: false
});
const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,
projectId
});
const actorOrgId = secretSync.connection.orgId;
const decryptSecretValue = (value?: Buffer | null) =>
value ? secretManagerDecryptor({ cipherTextBlob: value }).toString() : "";
const { expandSecretReferences } = expandSecretReferencesFactory({
decryptSecretValue,
secretDAL: secretV2BridgeDAL,
folderDAL,
projectId,
canExpandValue: () => true,
actorOrgId,
orgDAL,
projectFolderGrantDAL,
projectDAL,
kmsService
});
return buildSyncPayload(
{
projectId,
environment: environment.slug,
sourcePath: folder.path,
sourceFolderId: folderId,
syncOptions: secretSync.syncOptions as TSecretSync["syncOptions"],
includeImports
},
{
folderDAL,
projectEnvDAL,
secretV2BridgeDAL,
secretImportDAL,
expandSecretReferences,
decryptSecretValue,
fnSecretsV2FromImportsDeps: {
projectFolderGrantDAL,
actorOrgId,
orgDAL,
kmsService
}
}
);
};
const queueSecretSyncSyncSecretsById = async (payload: TQueueSecretSyncSyncSecretsByIdDTO) => {
const { attempts, backoff } = DEFAULT_SECRET_SYNC_RETRY_CONFIG;
return queueService.queue(QueueName.AppConnectionSecretSync, QueueJobs.SecretSyncSyncSecrets, payload, {
delay: getRequeueDelay(payload.failedToAcquireLockCount),
attempts,
backoff,
removeOnComplete: true,
removeOnFail: true,
jobId: randomUUID()
});
};
const queueSecretSyncImportSecretsById = async (payload: TQueueSecretSyncImportSecretsByIdDTO) =>
queueService.queue(QueueName.AppConnectionSecretSync, QueueJobs.SecretSyncImportSecrets, payload, {
attempts: 1,
removeOnComplete: true,
removeOnFail: true,
jobId: randomUUID()
});
const queueSecretSyncRemoveSecretsById = async (payload: TQueueSecretSyncRemoveSecretsByIdDTO) =>
queueService.queue(QueueName.AppConnectionSecretSync, QueueJobs.SecretSyncRemoveSecrets, payload, {
attempts: 1,
removeOnComplete: true,
removeOnFail: true,
jobId: randomUUID()
});
const $queueSendSecretSyncFailedNotifications = async (payload: TQueueSendSecretSyncActionFailedNotificationsDTO) => {
if (!appCfg.isSmtpConfigured) return;
await queueService.queue(
QueueName.AppConnectionSecretSync,
QueueJobs.SecretSyncSendActionFailedNotifications,
payload,
{
jobId: `secret-sync-${payload.secretSync.id}-failed-notifications`,
attempts: 5,
delay: 1000 * 60,
backoff: {
type: "exponential",
delay: 3000
},
removeOnFail: true,
removeOnComplete: true
}
);
};
const $importSecrets = async (
secretSync: TSecretSyncWithCredentials,
importBehavior: SecretSyncImportBehavior
): Promise<TSecretMap> => {
const {
projectId,
environment,
folder,
destination,
connection: { orgId }
} = secretSync;
await enterpriseSyncCheck(
licenseService,
destination,
orgId,
"Failed to import secrets due to plan restriction. Upgrade plan to access enterprise secret syncs."
);
if (!environment || !folder)
throw new Error(
"Invalid Secret Sync source configuration: folder no longer exists. Please update source environment and secret path."
);
const importedSecrets = await SecretSyncFns.getSecrets(secretSync, {
appConnectionDAL,
gitHubAppDAL,
kmsService,
gatewayV2Service,
gatewayPoolService,
oidcIssuerService
});
if (!Object.keys(importedSecrets).length) return {};
let invalidNameCount = 0;
let errorMessage = "";
for (const [key] of Object.entries(importedSecrets)) {
const result = SecretNameSchema.safeParse(key);
if (!result.success) {
invalidNameCount += 1;
if (errorMessage === "") errorMessage = result.error.issues[0]?.message;
}
}
if (invalidNameCount > 0) {
throw new SecretSyncError({
message: `Found ${invalidNameCount} invalid secret name${invalidNameCount === 1 ? "" : "s"}. ${errorMessage}`,
shouldRetry: false
});
}
const importedSecretMap: TSecretMap = {};
// Import behavior is never combined with including subfolders, so this payload always covers
// exactly one folder. Comparison is against raw keys as returned by the destination
// (already schema-stripped), so entries come from `secrets` rather than the schema-applying flatten().
const payload = await $getInfisicalSecrets(secretSync, false);
const secretMap: TSecretMap = Object.fromEntries(
payload.secrets.map((entry) => [
entry.key,
{ value: entry.value, id: entry.id, comment: entry.comment, secretMetadata: entry.secretMetadata }
])
);
const secretsToCreate: Parameters<typeof $createManySecretsRawFn>[0]["secrets"] = [];
const secretsToUpdate: Parameters<typeof $updateManySecretsRawFn>[0]["secrets"] = [];
Object.entries(importedSecrets).forEach(([key, secretData]) => {
const { value, comment = "", skipMultilineEncoding } = secretData;
const importKey =
destination === SecretSync.AzureKeyVault ? resolveAzureKeyVaultImportedSecretKey(key, secretMap) : key;
const secret = {
secretName: importKey,
secretValue: value,
type: SecretType.Shared,
secretComment: comment,
skipMultilineEncoding: skipMultilineEncoding ?? undefined
};
if (Object.hasOwn(secretMap, importKey)) {
// Only update secrets if the source value is not empty
if (value && value !== secretMap[importKey].value) {
secretsToUpdate.push(secret);
if (importBehavior === SecretSyncImportBehavior.PrioritizeDestination)
importedSecretMap[importKey] = secretData;
}
} else {
secretsToCreate.push(secret);
importedSecretMap[importKey] = secretData;
}
});
if (secretsToCreate.length) {
await $createManySecretsRawFn({
projectId,
path: folder.path,
environment: environment.slug,
secrets: secretsToCreate
});
}
if (importBehavior === SecretSyncImportBehavior.PrioritizeDestination && secretsToUpdate.length) {
await $updateManySecretsRawFn({
projectId,
path: folder.path,
environment: environment.slug,
secrets: secretsToUpdate
});
}
if (secretsToUpdate.length || secretsToCreate.length)
await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId);
return importedSecretMap;
};
const $handleSyncSecretsJob = async (job: TSecretSyncSyncSecretsDTO, secretSync: TSecretSyncRaw) => {
const {
data: { syncId, auditLogInfo }
} = job;
await enterpriseSyncCheck(
licenseService,
secretSync.destination as SecretSync,
secretSync.connection.orgId,
"Failed to sync secrets due to plan restriction. Upgrade plan to access enterprise secret syncs."
);
await secretSyncDAL.updateById(
syncId,
{
syncStatus: SecretSyncStatus.Running
},
{ includeFolderPath: false }
);
logger.info(
`SecretSync Sync [syncId=${secretSync.id}] [destination=${secretSync.destination}] [projectId=${secretSync.projectId}] [folderId=${secretSync.folderId}] [connectionId=${secretSync.connectionId}]`
);
let isSynced = false;
let syncMessage: string | null = null;
let isFinalAttempt = job.attemptsStarted === job.opts.attempts;
let syncResult: TSyncSecretsResult | undefined;
try {
const {
connection: { orgId, encryptedCredentials, projectId }
} = secretSync;
const credentials = await decryptAppConnectionCredentials({
orgId,
encryptedCredentials,
kmsService,
projectId
});
const secretSyncWithCredentials = {
...secretSync,
connection: {
...secretSync.connection,
credentials
}
} as TSecretSyncWithCredentials;
const {
lastSyncedAt,
syncOptions: { initialSyncBehavior }
} = secretSyncWithCredentials;
let payload = await $getInfisicalSecrets(secretSync);
if (!lastSyncedAt && initialSyncBehavior !== SecretSyncInitialSyncBehavior.OverwriteDestination) {
await $importSecrets(
secretSyncWithCredentials,
initialSyncBehavior === SecretSyncInitialSyncBehavior.ImportPrioritizeSource
? SecretSyncImportBehavior.PrioritizeSource
: SecretSyncImportBehavior.PrioritizeDestination
);
// $importSecrets writes directly to the source folder, so the payload built above is
// stale. Re-read rather than merge in memory, since the payload is no longer a plain map.
payload = await $getInfisicalSecrets(secretSync);
}
const result = await SecretSyncFns.syncSecrets(secretSyncWithCredentials, payload, {
appConnectionDAL,
gitHubAppDAL,
kmsService,
gatewayV2Service,
gatewayPoolService,
oidcIssuerService,
keyStore
});
syncResult = result ?? undefined;
syncMessage = syncResult?.message ?? null;
isSynced = true;
} catch (err) {
const axiosError = err instanceof SecretSyncError && isAxiosError(err.error) ? err.error : err;
logger.error(
isAxiosError(axiosError) ? axiosError.response?.data : err,
`SecretSync Sync Error [syncId=${secretSync.id}] [destination=${secretSync.destination}] [projectId=${secretSync.projectId}] [folderId=${secretSync.folderId}] [connectionId=${secretSync.connectionId}]`
);
if (appCfg.OTEL_TELEMETRY_COLLECTION_ENABLED) {
syncSecretsErrorHistogram.record(1, {
version: 1,
destination: secretSync.destination,
syncId: secretSync.id,
projectId: secretSync.projectId,
type: err instanceof AxiosError ? "AxiosError" : err?.constructor?.name || "UnknownError",
status: err instanceof AxiosError ? err.response?.status : undefined,
name: err instanceof Error ? err.name : undefined
});
}
syncMessage = parseSyncErrorMessage(err);
if (err instanceof SecretSyncError && !err.shouldRetry) {
isFinalAttempt = true;
} else {
// re-throw so job fails
throw err;
}
} finally {
const ranAt = new Date();
const syncStatus = isSynced ? SecretSyncStatus.Succeeded : SecretSyncStatus.Failed;
recordSecretSyncOutcomeMetric({
destination: secretSync.destination,
operation: "sync",
outcome: isSynced ? "success" : "failure",
attemptsExhausted: isFinalAttempt
});
await auditLogService.createAuditLog({
projectId: secretSync.projectId,
...(auditLogInfo ?? {
actor: {
type: ActorType.PLATFORM,
metadata: {}
}
}),
event: {
type: EventType.SECRET_SYNC_SYNC_SECRETS,
metadata: {
syncId: secretSync.id,
syncOptions: secretSync.syncOptions,
destination: secretSync.destination,
destinationConfig: secretSync.destinationConfig,
folderId: secretSync.folderId,
connectionId: secretSync.connectionId,
jobRanAt: ranAt,
jobId: job.id!,
syncStatus,
syncMessage,
createdSecretKeys: syncResult?.createdSecretKeys,
updatedSecretKeys: syncResult?.updatedSecretKeys,
deletedSecretKeys: syncResult?.deletedSecretKeys
}
}
});
if (isSynced || isFinalAttempt) {
// lastSyncResult is written only here, so lock and admission failures cannot overwrite the provider's result.
// The conditional update makes only the run that actually changes it report the change, whatever state this
// job loaded before taking the lock. A sync that never succeeded (null) failing is a new misconfiguration, not
// a breakage, so it is not counted.
const isStatusChange =
(
await secretSyncDAL.update(
{
id: secretSync.id,
lastSyncResult: isSynced ? SecretSyncStatus.Failed : SecretSyncStatus.Succeeded
},
{ lastSyncResult: syncStatus }
)
).length > 0;
if (isStatusChange) {
recordSecretSyncStatusChangeMetric({
destination: secretSync.destination,
newStatus: isSynced ? "healthy" : "failed"
});
}
const updatedSecretSync = await secretSyncDAL.updateById(
secretSync.id,
{
syncStatus,
lastSyncJobId: job.id,
lastSyncMessage: syncMessage,
lastSyncedAt: isSynced ? ranAt : undefined,
lastSyncResult: syncStatus
},
{ includeFolderPath: false }
);
if (!isSynced) {
void telemetryService
.sendPostHogEvents({
event: PostHogEventTypes.SecretSyncFailed,
distinctId: `platform/${secretSync.projectId}`,
organizationId: secretSync.connection.orgId,
properties: {
syncId: secretSync.id,
syncDestination: secretSync.destination,
projectId: secretSync.projectId
}
})
.catch(() => {});
await $queueSendSecretSyncFailedNotifications({
secretSync: { ...updatedSecretSync, folder: secretSync.folder },
action: SecretSyncAction.SyncSecrets,
auditLogInfo
});
}
}
}
logger.info("SecretSync Sync Job with ID %s Completed", job.id);
};
const $handleImportSecretsJob = async (job: TSecretSyncImportSecretsDTO, secretSync: TSecretSyncRaw) => {
const {
data: { syncId, auditLogInfo, importBehavior }
} = job;
await secretSyncDAL.update({ id: syncId }, { importStatus: SecretSyncStatus.Running });
logger.info(
`SecretSync Import [syncId=${secretSync.id}] [destination=${secretSync.destination}] [projectId=${secretSync.projectId}] [folderId=${secretSync.folderId}] [connectionId=${secretSync.connectionId}]`
);
let isSuccess = false;
let importMessage: string | null = null;
const isFinalAttempt = job.attemptsStarted === job.opts.attempts;
try {
const {
connection: { orgId, encryptedCredentials, projectId }
} = secretSync;
const credentials = await decryptAppConnectionCredentials({
orgId,
encryptedCredentials,
kmsService,
projectId
});
await $importSecrets(
{
...secretSync,
connection: {
...secretSync.connection,
credentials
}
} as TSecretSyncWithCredentials,
importBehavior
);
isSuccess = true;
} catch (err) {
logger.error(
err,
`SecretSync Import Error [syncId=${secretSync.id}] [destination=${secretSync.destination}] [projectId=${secretSync.projectId}] [folderId=${secretSync.folderId}] [connectionId=${secretSync.connectionId}]`
);
if (appCfg.OTEL_TELEMETRY_COLLECTION_ENABLED) {
importSecretsErrorHistogram.record(1, {
version: 1,
destination: secretSync.destination,
syncId: secretSync.id,
projectId: secretSync.projectId,
type: err instanceof AxiosError ? "AxiosError" : err?.constructor?.name || "UnknownError",
status: err instanceof AxiosError ? err.response?.status : undefined,
name: err instanceof Error ? err.name : undefined
});
}
importMessage = parseSyncErrorMessage(err);
// re-throw so job fails
throw err;
} finally {
const ranAt = new Date();
const importStatus = isSuccess ? SecretSyncStatus.Succeeded : SecretSyncStatus.Failed;
recordSecretSyncOutcomeMetric({
destination: secretSync.destination,
operation: "import",
outcome: isSuccess ? "success" : "failure",
attemptsExhausted: isFinalAttempt
});
await auditLogService.createAuditLog({
projectId: secretSync.projectId,
...(auditLogInfo ?? {
actor: {
type: ActorType.PLATFORM,
metadata: {}
}
}),
event: {
type: EventType.SECRET_SYNC_IMPORT_SECRETS,
metadata: {
syncId: secretSync.id,
syncOptions: secretSync.syncOptions,
destination: secretSync.destination,
destinationConfig: secretSync.destinationConfig,
folderId: secretSync.folderId,
connectionId: secretSync.connectionId,
jobRanAt: ranAt,
jobId: job.id!,
importStatus,
importMessage,
importBehavior
}
}
});
if (isSuccess || isFinalAttempt) {
const updatedSecretSync = await secretSyncDAL.updateById(
secretSync.id,
{
importStatus,
lastImportJobId: job.id,
lastImportMessage: importMessage,
lastImportedAt: isSuccess ? ranAt : undefined
},
{ includeFolderPath: false }
);
if (!isSuccess) {
await $queueSendSecretSyncFailedNotifications({
secretSync: { ...updatedSecretSync, folder: secretSync.folder },
action: SecretSyncAction.ImportSecrets,
auditLogInfo
});
}
}
}
logger.info("SecretSync Import Job with ID %s Completed", job.id);
};
const $handleRemoveSecretsJob = async (job: TSecretSyncRemoveSecretsDTO, secretSync: TSecretSyncRaw) => {
const {
data: { syncId, auditLogInfo, deleteSyncOnComplete }
} = job;
await enterpriseSyncCheck(
licenseService,
secretSync.destination as SecretSync,
secretSync.connection.orgId,
"Failed to remove secrets due to plan restriction. Upgrade plan to access enterprise secret syncs."
);
await secretSyncDAL.updateById(
syncId,
{
removeStatus: SecretSyncStatus.Running
},
{ includeFolderPath: false }
);
logger.info(
`SecretSync Remove [syncId=${secretSync.id}] [destination=${secretSync.destination}] [projectId=${secretSync.projectId}] [folderId=${secretSync.folderId}] [connectionId=${secretSync.connectionId}]`
);
let isSuccess = false;
let removeMessage: string | null = null;
const isFinalAttempt = job.attemptsStarted === job.opts.attempts;
try {
const {
connection: { orgId, encryptedCredentials, projectId }
} = secretSync;
const credentials = await decryptAppConnectionCredentials({
orgId,
encryptedCredentials,
kmsService,
projectId
});
const payload = await $getInfisicalSecrets(secretSync, true);
await SecretSyncFns.removeSecrets(
{
...secretSync,
connection: {
...secretSync.connection,
credentials
}
} as TSecretSyncWithCredentials,
// We don't care about duplicates when removing secrets: a duplicate name still maps to
// the same destination key we're deleting either way.
payload.dedupeConflicts(),
{
appConnectionDAL,
gitHubAppDAL,
kmsService,
gatewayV2Service,
gatewayPoolService,
oidcIssuerService
}
);
isSuccess = true;
} catch (err) {
logger.error(
err,
`SecretSync Remove Error [syncId=${secretSync.id}] [destination=${secretSync.destination}] [projectId=${secretSync.projectId}] [folderId=${secretSync.folderId}] [connectionId=${secretSync.connectionId}]`
);
if (appCfg.OTEL_TELEMETRY_COLLECTION_ENABLED) {
removeSecretsErrorHistogram.record(1, {
version: 1,
destination: secretSync.destination,
syncId: secretSync.id,
projectId: secretSync.projectId,
type: err instanceof AxiosError ? "AxiosError" : err?.constructor?.name || "UnknownError",
status: err instanceof AxiosError ? err.response?.status : undefined,
name: err instanceof Error ? err.name : undefined
});
}
removeMessage = parseSyncErrorMessage(err);
// re-throw so job fails
throw err;
} finally {
const ranAt = new Date();
const removeStatus = isSuccess ? SecretSyncStatus.Succeeded : SecretSyncStatus.Failed;
recordSecretSyncOutcomeMetric({
destination: secretSync.destination,
operation: "remove",
outcome: isSuccess ? "success" : "failure",
attemptsExhausted: isFinalAttempt
});
await auditLogService.createAuditLog({
projectId: secretSync.projectId,
...(auditLogInfo ?? {
actor: {
type: ActorType.PLATFORM,
metadata: {}
}
}),
event: {
type: EventType.SECRET_SYNC_REMOVE_SECRETS,
metadata: {
syncId: secretSync.id,
syncOptions: secretSync.syncOptions,
destination: secretSync.destination,
destinationConfig: secretSync.destinationConfig,
folderId: secretSync.folderId,
connectionId: secretSync.connectionId,
jobRanAt: ranAt,
jobId: job.id!,
removeStatus,
removeMessage
}
}
});
if (isSuccess || isFinalAttempt) {
if (isSuccess && deleteSyncOnComplete) {
await secretSyncDAL.deleteById(secretSync.id);
} else {
const updatedSecretSync = await secretSyncDAL.updateById(
secretSync.id,
{
removeStatus,
lastRemoveJobId: job.id,
lastRemoveMessage: removeMessage,
lastRemovedAt: isSuccess ? ranAt : undefined
},
{ includeFolderPath: false }
);
if (!isSuccess) {
await $queueSendSecretSyncFailedNotifications({
secretSync: { ...updatedSecretSync, folder: secretSync.folder },
action: SecretSyncAction.RemoveSecrets,
auditLogInfo
});
}
}
}
}
logger.info("SecretSync Remove Job with ID %s Completed", job.id);
};
const $sendSecretSyncFailedNotifications = async (job: TSendSecretSyncFailedNotificationsJobDTO) => {
const {
data: { secretSync, auditLogInfo, action }
} = job;
const { projectId, destination, name, folder, lastSyncMessage, lastRemoveMessage, lastImportMessage, environment } =
secretSync;
const projectMembers = await projectMembershipDAL.findAllProjectMembers(projectId);
const project = await projectDAL.findById(projectId);
let projectAdmins = projectMembers.filter((member) =>
member.roles.some((role) => role.role === ProjectMembershipRole.Admin)
);
const triggeredByUserId =
auditLogInfo && auditLogInfo.actor.type === ActorType.USER && auditLogInfo.actor.metadata.userId;
// only notify triggering user if triggered by admin
if (triggeredByUserId && projectAdmins.map((admin) => admin.userId).includes(triggeredByUserId)) {
projectAdmins = projectAdmins.filter((admin) => admin.userId === triggeredByUserId);
}
const syncDestination = SECRET_SYNC_NAME_MAP[destination as SecretSync];
let actionLabel: string;
let failureMessage: string | null | undefined;
switch (action) {
case SecretSyncAction.ImportSecrets:
actionLabel = "Import";
failureMessage = lastImportMessage;
break;
case SecretSyncAction.RemoveSecrets:
actionLabel = "Remove";
failureMessage = lastRemoveMessage;
break;
case SecretSyncAction.SyncSecrets:
default:
actionLabel = `Sync`;
failureMessage = lastSyncMessage;
break;
}
const baseProjectPath = `/organizations/${project.orgId}/projects/secret-management/${projectId}`;
const overviewPath = `${baseProjectPath}/overview`;
const syncPath = `${baseProjectPath}/integrations/secret-syncs/${destination}/${secretSync.id}`;
const notifications = [
triggerWorkflowIntegrationNotification({
input: {
notification: {
type: TriggerFeature.SECRET_SYNC_ERROR,
payload: {
syncName: name,
syncDestination,
failureMessage: failureMessage || "An unknown error occurred",
syncUrl: `${appCfg.SITE_URL}${syncPath}`,
syncActionLabel: actionLabel,
environment: environment?.name || "-",
secretPath: folder?.path || "-",
projectName: project.name,
projectPath: overviewPath
}
},
projectId
},
dependencies: {
projectDAL,
projectSlackConfigDAL,
kmsService,
microsoftTeamsService,
projectMicrosoftTeamsConfigDAL
}
}),
notificationService.createUserNotifications(
projectAdmins.map((admin) => ({
userId: admin.userId,
orgId: project.orgId,
type: NotificationType.SECRET_SYNC_FAILED,
title: `Secret Sync Failed to ${actionLabel} Secrets`,
body: `Your **${syncDestination}** sync **${name}** failed to complete${failureMessage ? `: \`${failureMessage}\`` : ""}`,
link: syncPath
}))
),
smtpService.sendMail({
recipients: projectAdmins.map((member) => member.user.email!).filter(Boolean),
template: SmtpTemplates.SecretSyncFailed,
subjectLine: `Secret Sync Failed to ${actionLabel} Secrets`,
substitutions: {
syncName: name,
syncDestination,
content: `Your ${syncDestination} Sync named "${name}" failed while attempting to ${action.toLowerCase()} secrets.`,
failureMessage,
secretPath: folder?.path,
environment: environment?.name,
projectName: project.name,
syncUrl: `${appCfg.SITE_URL}${syncPath}`
}
})
];
await Promise.allSettled(notifications);
};
// Takes a path rather than a folder, so it serves a write inside a folder and the deletion of the
// folder itself. The path need not still exist: the only thing the folder is needed for is the
// sync rooted exactly on it, and a deleted folder has no such sync left to find, because
// secret_syncs.folderId is SET NULL by its foreign key.
const queueSecretSyncsSyncSecretsByPath = async ({
secretPath,
projectId,
environmentSlug
}: TQueueSecretSyncsByPathDTO) => {
const environment = await projectEnvDAL.findOne({ projectId, slug: environmentSlug });
if (!environment) return;
const folder = await folderDAL.findBySecretPath(projectId, environmentSlug, secretPath);
const ancestorFolders = (
await folderDAL.findByManySecretPath(
getAncestorPaths(secretPath).map((path) => ({ envId: environment.id, secretPath: path }))
)
).filter((entry): entry is NonNullable<typeof entry> => Boolean(entry));
const candidateSyncs = await secretSyncDAL.find(
{
$in: { folderId: [...ancestorFolders.map((entry) => entry.id), ...(folder ? [folder.id] : [])] },
isAutoSyncEnabled: true
},
undefined,
{ includeFolderPath: false }
);
// A sync on the path itself always matches, whether or not it includes subfolders. A sync on an
// ancestor folder only matches when it includes them, so a sync rooted above this path that does
// not is never triggered by a change it was never configured to cover.
// The move and copy warnings in frontend/src/helpers/secretSyncCoverage.ts mirror this rule.
const secretSyncs = candidateSyncs.filter(
(sync) =>
(folder && sync.folderId === folder.id) ||
Boolean((sync.syncOptions as TSecretSync["syncOptions"])?.includeAllSubFolders)
);
await secretSyncDAL.update(
{
$in: {
id: secretSyncs.map((sync) => sync.id)
}
},
{
syncStatus: SecretSyncStatus.Pending
}
);
await Promise.all(
secretSyncs.map((secretSync) =>
queueSecretSyncSyncSecretsById({
syncId: secretSync.id
})
)
);
};
const $handleAcquireLockFailure = async (job: SecretSyncActionJob) => {
const { syncId, auditLogInfo } = job.data;
switch (job.name) {
case QueueJobs.SecretSyncSyncSecrets: {
const { failedToAcquireLockCount = 0, ...rest } = job.data as TQueueSecretSyncSyncSecretsByIdDTO;
if (failedToAcquireLockCount < REQUEUE_LIMIT) {
await queueSecretSyncSyncSecretsById({ ...rest, failedToAcquireLockCount: failedToAcquireLockCount + 1 });
return;
}
const secretSync = await secretSyncDAL.updateById(syncId, {
syncStatus: SecretSyncStatus.Failed,
lastSyncMessage:
"Failed to run job. This typically happens when a sync is already in progress. Please try again.",
lastSyncJobId: job.id
});
recordSecretSyncOutcomeMetric({
destination: secretSync.destination,
operation: "sync",
outcome: "failure",
attemptsExhausted: true
});
await $queueSendSecretSyncFailedNotifications({
secretSync,
action: SecretSyncAction.SyncSecrets,
auditLogInfo
});
break;
}
// Scott: the two cases below are unlikely to happen as we check the lock at the API level but including this as a fallback
case QueueJobs.SecretSyncImportSecrets: {
const secretSync = await secretSyncDAL.updateById(syncId, {
importStatus: SecretSyncStatus.Failed,
lastImportMessage:
"Failed to run job. This typically happens when a sync is already in progress. Please try again.",
lastImportJobId: job.id
});
recordSecretSyncOutcomeMetric({
destination: secretSync.destination,
operation: "import",
outcome: "failure",
attemptsExhausted: true
});
await $queueSendSecretSyncFailedNotifications({
secretSync,
action: SecretSyncAction.ImportSecrets,
auditLogInfo
});
break;
}
case QueueJobs.SecretSyncRemoveSecrets: {
const secretSync = await secretSyncDAL.updateById(syncId, {
removeStatus: SecretSyncStatus.Failed,
lastRemoveMessage:
"Failed to run job. This typically happens when a sync is already in progress. Please try again.",
lastRemoveJobId: job.id
});
recordSecretSyncOutcomeMetric({
destination: secretSync.destination,
operation: "remove",
outcome: "failure",
attemptsExhausted: true
});
await $queueSendSecretSyncFailedNotifications({
secretSync,
action: SecretSyncAction.RemoveSecrets,
auditLogInfo
});
break;
}
default:
// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
throw new Error(`Unhandled Secret Sync Job ${job.name}`);
}
};
const $handleDailySecretSyncRetry = async () => {
const updatedIds = await secretSyncDAL.updateAndReturnIds(
{
syncStatus: SecretSyncStatus.Failed,
isAutoSyncEnabled: true,
$in: { destination: [...SECRET_SYNC_DAILY_RETRY_DESTINATIONS] }
},
{ syncStatus: SecretSyncStatus.Pending }
);
if (!updatedIds.length) return;
await Promise.all(
updatedIds.map((syncId) =>
queueSecretSyncSyncSecretsById({
syncId
})
)
);
};
queueService.start(QueueName.AppConnectionSecretSync, async (job) => {
if (job.name === QueueJobs.SecretSyncSendActionFailedNotifications) {
await $sendSecretSyncFailedNotifications(job as TSendSecretSyncFailedNotificationsJobDTO);
return;
}
if (job.name === QueueJobs.DailySecretSyncRetry) {
await $handleDailySecretSyncRetry();
return;
}
const { syncId } = job.data as
| TQueueSecretSyncSyncSecretsByIdDTO
| TQueueSecretSyncImportSecretsByIdDTO
| TQueueSecretSyncRemoveSecretsByIdDTO;
const connectionId = await secretSyncDAL.findConnectionId(syncId);
if (!connectionId) {
// skip rather than throw, so it doesn't retry-storm when a sync is deleted.
logger.info(`AppConnectionSecretSync: secret sync ${syncId} not found, skipping`);
return;
}
let lock: Awaited<ReturnType<typeof keyStore.acquireLock>>;
try {
lock = await keyStore.acquireLock(
[KeyStorePrefixes.SecretSyncLock(syncId)],
// scott: not sure on this duration; syncs can take excessive amounts of time so we need to keep it locked,
// but should always release below...
5 * 60 * 1000
);
} catch (e) {
logger.info(`SecretSync Failed to acquire lock [syncId=${syncId}] [job=${job.name}]`);
await $handleAcquireLockFailure(job as SecretSyncActionJob);
return;
}
let admittedConnectionSlot = false;
try {
if (job.name === QueueJobs.SecretSyncSyncSecrets) {
admittedConnectionSlot = await $tryAdmitConnectionConcurrency(connectionId);
if (!admittedConnectionSlot) {
logger.info(
`SecretSync Concurrency limit reached [syncId=${syncId}] [job=${job.name}] [connectionId=${connectionId}]`
);
await $handleAcquireLockFailure(job as SecretSyncActionJob);
return;
}
}
const secretSync = await secretSyncDAL.findById(syncId);
if (!secretSync) {
logger.info(`AppConnectionSecretSync: secret sync ${syncId} not found, skipping`);
return;
}
if (secretSync.connectionId !== connectionId && job.name === QueueJobs.SecretSyncSyncSecrets) {
logger.info(
`SecretSync connection changed, releasing slot and requeueing [syncId=${syncId}] [job=${job.name}] [connectionId=${connectionId}] [currentConnectionId=${secretSync.connectionId}]`
);
await $handleAcquireLockFailure(job as SecretSyncActionJob);
return;
}
switch (job.name) {
case QueueJobs.SecretSyncSyncSecrets: {
await $handleSyncSecretsJob(job as TSecretSyncSyncSecretsDTO, secretSync);
break;
}
case QueueJobs.SecretSyncImportSecrets:
await $handleImportSecretsJob(job as TSecretSyncImportSecretsDTO, secretSync);
break;
case QueueJobs.SecretSyncRemoveSecrets:
await $handleRemoveSecretsJob(job as TSecretSyncRemoveSecretsDTO, secretSync);
break;
default:
// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
throw new Error(`Unhandled Secret Sync Job ${job.name}`);
}
} finally {
if (admittedConnectionSlot) {
await $releaseConnectionConcurrency(connectionId);
}
await lock.release();
}
});
const startDailySecretSyncRetryJob = () => {
cronJob.register({
name: CronJobName.DailySecretSyncRetry,
pattern: appCfg.isDailyResourceCleanUpDevelopmentMode ? "*/5 * * * *" : "0 0 * * *",
runHashTtlS: 3 * 24 * 60 * 60,
handler: async () => {
await queueService.queue(
QueueName.AppConnectionSecretSync,
QueueJobs.DailySecretSyncRetry,
undefined as never,
{ jobId: CronJobName.DailySecretSyncRetry }
);
}
});
};
return {
queueSecretSyncSyncSecretsById,
queueSecretSyncImportSecretsById,
queueSecretSyncRemoveSecretsById,
queueSecretSyncsSyncSecretsByPath,
startDailySecretSyncRetryJob
};
};