Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.
TypeScript acmeagentsai-agentscertificate-managementdevsecopsenvironment-variableskubernetesopen-sourcepampkipostgresprivate-caprivileged-access-managementsecret-managementsecret-managersecretssecrets-managementsecuritysecurity-toolsvault
import pLimit from "p-limit";
import { TAlertChannels, TAlerts } from "@app/db/schemas";
import { logger } from "@app/lib/logger";
import { AlertDispatchOutcome } from "@app/lib/telemetry/metrics";
import { TKmsServiceFactory } from "@app/services/kms/kms-service";
import { TSmtpService } from "@app/services/smtp/smtp-service";
import { decryptChannelConfig, getAlertChannelCipher } from "./alert-channel-crypto-fns";
import { TAlertChannelDALFactory } from "./alert-channel-dal";
import { TAlertChannelRecipientDALFactory } from "./alert-channel-recipient-dal";
import { AlertChannelType, TAlertChannelDeps, TAlertRecipient, TChannelTargetResult } from "./alert-channel-types";
import { getDedupCutoff, TAlertHistoryDALFactory } from "./alert-history-dal";
import { getRecipientScope } from "./alert-principal-scope-fns";
import { TAlertProviderRegistry } from "./alert-provider-registry";
import { TAlertRecipientResolver } from "./alert-recipient-resolver";
import { AlertRunStatus, DEFAULT_DEDUP_WINDOW_HOURS, IResourceAlertProvider, TAlertContext } from "./alert-types";
import { ALERT_CHANNEL_REGISTRY } from "./channels/alert-channel-registry";
const ALERT_DELIVERY_CONCURRENCY = 10;
const HISTORY_WRITE_ATTEMPTS = 3;
const HISTORY_WRITE_RETRY_DELAY_MS = 250;
const sleep = (ms: number) =>
new Promise<void>((resolve) => {
setTimeout(resolve, ms);
});
type TChannelWork = {
channel: TAlertChannels;
due: { target: unknown; id: string }[];
};
type TChannelDispatchResult = {
channelId: string;
channelType: string;
targetIds: string[];
success: boolean;
error?: string;
targetResults?: TChannelTargetResult[];
skipped?: boolean;
};
export type TAlertEngineDep = {
alertChannelDAL: Pick<TAlertChannelDALFactory, "findByAlertId">;
alertChannelRecipientDAL: Pick<TAlertChannelRecipientDALFactory, "findByChannelIds">;
alertHistoryDAL: Pick<
TAlertHistoryDALFactory,
"createWithTargets" | "findRecentlyAlertedTargets" | "findDeliveredChannelIdsForEvent"
>;
alertProviderRegistry: TAlertProviderRegistry;
alertRecipientResolver: Pick<TAlertRecipientResolver, "resolveMany">;
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey">;
smtpService: Pick<TSmtpService, "sendMail">;
};
export type TAlertEngine = ReturnType<typeof alertEngineFactory>;
export const alertEngineFactory = ({
alertChannelDAL,
alertChannelRecipientDAL,
alertHistoryDAL,
alertProviderRegistry,
alertRecipientResolver,
kmsService,
smtpService
}: TAlertEngineDep) => {
const $dispatchChannelWork = async (
alert: TAlerts,
provider: IResourceAlertProvider,
channelWork: TChannelWork[],
eventId?: string
): Promise<AlertDispatchOutcome> => {
const alertContext: TAlertContext = {
id: alert.id,
name: alert.name,
orgId: alert.orgId,
projectId: alert.projectId,
resourceType: alert.resourceType,
resourceId: alert.resourceId,
eventType: alert.eventType,
condition: alert.condition
};
const viewUrl = await provider.buildViewUrl(alertContext);
const { decryptor } = await getAlertChannelCipher(kmsService, {
orgId: alert.orgId,
projectId: alert.projectId
});
const directedChannelIds = channelWork
.filter((work) => ALERT_CHANNEL_REGISTRY[work.channel.channelType as AlertChannelType]?.directed)
.map((work) => work.channel.id);
let recipientsByChannel = new Map<string, TAlertRecipient[]>();
if (directedChannelIds.length) {
const recipientRows = await alertChannelRecipientDAL.findByChannelIds(directedChannelIds);
const rowsByChannel = new Map<string, { principalType: string; principalId: string }[]>();
recipientRows.forEach((row) => {
const list = rowsByChannel.get(row.channelId) ?? [];
list.push(row);
rowsByChannel.set(row.channelId, list);
});
recipientsByChannel = await alertRecipientResolver.resolveMany(rowsByChannel, {
orgId: alert.orgId,
projectId: getRecipientScope(provider, alert.projectId).projectId
});
}
const deps: TAlertChannelDeps = { smtpService };
const sendLimit = pLimit(ALERT_DELIVERY_CONCURRENCY);
const channelResults = await Promise.all(
channelWork.map(async ({ channel, due }): Promise<TChannelDispatchResult> => {
const targetIds = due.map((target) => target.id);
const base = { channelId: channel.id, channelType: channel.channelType, targetIds };
const definition = ALERT_CHANNEL_REGISTRY[channel.channelType as AlertChannelType];
if (!definition) return { ...base, success: false, error: "Unknown channel type" };
let config: unknown;
try {
config = decryptChannelConfig(channel.encryptedConfig, decryptor);
} catch (err) {
logger.error(err, `Failed to decrypt alert channel config [channelId=${channel.id}]`);
return { ...base, success: false, error: "Failed to decrypt channel config" };
}
const payload = provider.buildPayload(
alertContext,
due.map((target) => target.target),
viewUrl
);
try {
if (definition.directed) {
const recipients = recipientsByChannel.get(channel.id) ?? [];
if (recipients.length === 0) {
logger.warn(
`Alert ${channel.channelType} channel has no resolvable recipients; skipping it this run [alertId=${alert.id}] [channelId=${channel.id}]`
);
return { ...base, success: false, skipped: true };
}
const results = await Promise.all(
recipients.map((recipient) =>
sendLimit(() => definition.send({ channelId: channel.id, config, payload, recipient, deps }))
)
);
const failures = results.filter((result) => !result.success);
if (failures.length === results.length) {
return { ...base, success: false, error: failures.map((f) => f.error).join("; ") };
}
if (failures.length > 0) {
logger.warn(
`Alert ${channel.channelType} channel had partial delivery failures [alertId=${alert.id}] [channelId=${channel.id}]: ${failures
.map((f) => f.error)
.join("; ")}`
);
}
return { ...base, success: true };
}
const result = await sendLimit(() => definition.send({ channelId: channel.id, config, payload, deps }));
return { ...base, ...result };
} catch (err) {
const error = err instanceof Error ? err.message : "Unknown error";
logger.error(err, `Failed to dispatch alert ${channel.channelType} channel [alertId=${alert.id}]`);
return { ...base, success: false, error };
}
})
);
const dispatched = channelResults.filter((result) => !result.skipped);
if (dispatched.length === 0) return AlertDispatchOutcome.NoRecipients;
const deliveries = dispatched.flatMap((result) => {
const perTarget = new Map<string, boolean>((result.targetResults ?? []).map((t) => [t.targetId, t.success]));
return result.targetIds.map((targetId) => ({
targetId,
channelId: result.channelId,
channelType: result.channelType,
status: (perTarget.get(targetId) ?? result.success) ? AlertRunStatus.SUCCESS : AlertRunStatus.FAILED
}));
});
const errors = dispatched
.filter((result) => !result.success && result.error)
.map((result) => `${result.channelType}: ${result.error}`);
const anyDelivered = dispatched.some((result) => result.success);
let status = AlertRunStatus.SUCCESS;
if (errors.length > 0) status = anyDelivered ? AlertRunStatus.PARTIAL : AlertRunStatus.FAILED;
const errorText = errors.length > 0 ? errors.join("\n") : undefined;
if (errorText) {
logger.error(`Alert delivery failed on one or more channels [alertId=${alert.id}]: ${errorText}`);
}
// Never fails the run: channels already sent, and throwing on the event path would re-notify.
for (let attempt = 1; attempt <= HISTORY_WRITE_ATTEMPTS; attempt += 1) {
try {
// eslint-disable-next-line no-await-in-loop -- retrying the same insert is the point
await alertHistoryDAL.createWithTargets(alert.id, { status, eventId }, deliveries);
break;
} catch (err) {
if (attempt === HISTORY_WRITE_ATTEMPTS) {
logger.error(err, `Failed to record alert history after delivery [alertId=${alert.id}]`);
break;
}
logger.warn(err, `Failed to record alert history, retrying [attempt=${attempt}] [alertId=${alert.id}]`);
// eslint-disable-next-line no-await-in-loop -- see above
await sleep(HISTORY_WRITE_RETRY_DELAY_MS * attempt);
}
}
if (status === AlertRunStatus.SUCCESS) return AlertDispatchOutcome.DeliverySuccess;
return status === AlertRunStatus.PARTIAL
? AlertDispatchOutcome.DeliveryPartial
: AlertDispatchOutcome.DeliveryFailed;
};
const $buildChannelWork = (
alert: TAlerts,
channels: TAlertChannels[],
targets: { target: unknown; id: string }[],
alreadyAlerted?: Set<string>
): TChannelWork[] =>
channels
.map((channel) => {
const due = alreadyAlerted
? targets.filter((target) => !alreadyAlerted.has(`${channel.id}:${target.id}`))
: targets;
const cap = ALERT_CHANNEL_REGISTRY[channel.channelType as AlertChannelType]?.maxTargetsPerRun;
if (!cap || due.length <= cap) return { channel, due };
const dropped = due.slice(cap);
if (alreadyAlerted) {
logger.info(
`Alert ${channel.channelType} channel capped at ${cap} targets this run; ${dropped.length} deferred to the next run [alertId=${alert.id}] [channelId=${channel.id}]`
);
} else {
logger.warn(
`Alert ${channel.channelType} channel caps at ${cap} targets; dropping ${dropped.length} from this event [alertId=${alert.id}] [channelId=${channel.id}] [dropped=${dropped.map((target) => target.id).join(",")}]`
);
}
return { channel, due: due.slice(0, cap) };
})
.filter((work) => work.due.length > 0);
const $getProvider = (alert: TAlerts, discovery: "findScheduledTargets" | "findEventTargets") => {
const provider = alertProviderRegistry.get(alert.resourceType);
if (!provider) {
logger.warn(`No alert provider registered for resource type '${alert.resourceType}' [alertId=${alert.id}]`);
return undefined;
}
if (!provider[discovery]) {
logger.warn(
`Alert provider '${alert.resourceType}' has no ${discovery}, so this alert cannot run [alertId=${alert.id}]`
);
return undefined;
}
return provider;
};
const runAlert = async (alert: TAlerts, opts?: { asOf?: Date }): Promise<AlertDispatchOutcome> => {
const provider = $getProvider(alert, "findScheduledTargets");
if (!provider?.findScheduledTargets) return AlertDispatchOutcome.NoProvider;
const channels = await alertChannelDAL.findByAlertId(alert.id, { enabled: true });
if (channels.length === 0) return AlertDispatchOutcome.NoChannels;
const window = provider.dedupWindowHours?.(alert.condition) ?? DEFAULT_DEDUP_WINDOW_HOURS;
const dueTargets = await provider.findScheduledTargets({
orgId: alert.orgId,
projectId: alert.projectId,
resourceId: alert.resourceId,
eventType: alert.eventType,
condition: alert.condition,
asOf: opts?.asOf ?? new Date(),
alreadyAlerted: {
alertId: alert.id,
channelIds: channels.map((channel) => channel.id),
since: getDedupCutoff(window)
}
});
if (dueTargets.length === 0) return AlertDispatchOutcome.NoDueTargets;
const targets = dueTargets.map((target) => ({ target, id: provider.targetId(target) }));
const recentlyAlerted = await alertHistoryDAL.findRecentlyAlertedTargets(
alert.id,
targets.map((target) => target.id),
window
);
const alertedSet = new Set(recentlyAlerted.map((row) => `${row.channelId}:${row.targetId}`));
const channelWork = $buildChannelWork(alert, channels, targets, alertedSet);
if (channelWork.length === 0) return AlertDispatchOutcome.AllDeduped;
return $dispatchChannelWork(alert, provider, channelWork);
};
const runAlertForEvent = async (
alert: TAlerts,
input: {
eventId: string;
eventType: string;
occurredAt: Date;
targetIds: string[];
payload: Record<string, unknown>;
}
): Promise<AlertDispatchOutcome> => {
const provider = $getProvider(alert, "findEventTargets");
if (!provider?.findEventTargets) return AlertDispatchOutcome.NoProvider;
const [enabledChannels, alreadyDelivered] = await Promise.all([
alertChannelDAL.findByAlertId(alert.id, { enabled: true, readFromPrimary: true }),
alertHistoryDAL.findDeliveredChannelIdsForEvent(alert.id, input.eventId)
]);
const skip = new Set(alreadyDelivered);
const channels = enabledChannels.filter(
(channel) => !skip.has(channel.id) && new Date(channel.createdAt) <= new Date(input.occurredAt)
);
if (channels.length === 0) return AlertDispatchOutcome.NoChannels;
const resolved = await provider.findEventTargets({
orgId: alert.orgId,
projectId: alert.projectId,
resourceId: alert.resourceId,
eventType: input.eventType,
condition: alert.condition,
targetIds: input.targetIds,
payload: input.payload
});
if (resolved.length === 0) return AlertDispatchOutcome.NoDueTargets;
const targets = resolved.map((target) => ({ target, id: provider.targetId(target) }));
return $dispatchChannelWork(alert, provider, $buildChannelWork(alert, channels, targets), input.eventId);
};
return { runAlert, runAlertForEvent };
};