🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
/* eslint-disable @typescript-eslint/no-use-before-define */
import {
InternalServerError,
PromptWebhookOutboundSchema,
SkillWebhookOutboundSchema,
WebhookDefaultHeaders,
ActionExecutionStatus,
LangfuseNotFoundError,
JobConfigState,
isSlackActionConfig,
isWebhookAction,
isGitHubDispatchAction,
isPagerDutyActionConfig,
isBaseError,
type AutomationDomain,
type ActionDomainWithSecrets,
type PagerDutyActionConfigWithSecrets,
} from "@langfuse/shared";
import { decrypt, createSignatureHeader } from "@langfuse/shared/encryption";
import { Prisma, prisma } from "@langfuse/shared/src/db";
import {
validateWebhookURL,
webhookWhitelistFromEnv,
whitelistFromEnv,
fetchWithSecureRedirects,
urlForLog,
type WebhookValidationWhitelist,
WEBHOOK_URL_VALIDATION_LOG_CONTEXT,
buildWebhookRequestHeaders,
recordIncrement,
} from "@langfuse/shared/src/server";
import {
TQueueJobTypes,
QueueName,
WebhookInput,
getAutomationById,
getActionByIdWithSecrets,
getActionById,
getConsecutiveAutomationFailures,
SlackService,
logger,
redis,
ProjectNotificationWebhookQueueEventSchema,
buildProjectNotificationSlackMessage,
} from "@langfuse/shared/src/server";
import { type MonitorAlert } from "@langfuse/shared/monitors";
import {
MonitorWebhookQueueEventSchema,
buildMonitorAlertSlackMessage,
} from "@langfuse/shared/monitors/server";
import {
pagerDutyClient,
type PagerDutyRegion,
} from "@langfuse/shared/pagerduty/server";
import { Processor, Job } from "bullmq";
import { backOff } from "exponential-backoff";
import { env } from "../env";
import { SlackMessageBuilder } from "../features/slack/slackMessageBuilder";
import {
buildPagerDutyEvent,
type PagerDutyEvent,
} from "../features/pagerduty/buildPagerDutyEvent";
/** WEBHOOK_ADDRESS_ATTEMPT_TIMEOUT_MS lets a retransmitted SYN finish before Node closes the socket. */
const WEBHOOK_ADDRESS_ATTEMPT_TIMEOUT_MS = 2_000;
// GitHub repository_dispatch client_payload: max 10 top-level properties and <64KB.
// https://docs.github.com/en/rest/repos/repos#create-a-repository-dispatch-event
const GITHUB_REPOSITORY_DISPATCH_MAX_PAYLOAD_BYTES = 64 * 1024;
const GITHUB_REPOSITORY_DISPATCH_TRUNCATION_MARKER =
"[TRUNCATED: GitHub repository_dispatch payload exceeded size limit]";
const GITHUB_REPOSITORY_DISPATCH_TRUNCATED_FIELDS = [
"prompt.prompt",
"prompt.config",
];
/** automationFailureThreshold is the consecutive-failure count after which a monitor-alert trigger is auto-disabled. Mirrors prompt-version's getConsecutiveAutomationFailures threshold (>= 5 failures since the last success). */
const automationFailureThreshold = 5;
const automationFailureTtlSeconds = 24 * 60 * 60;
const automationFailureKey = (projectId: string, automationId: string) =>
`automation-failures:${projectId}:${automationId}`;
/** incrementAutomationFailure bumps the consecutive-failure counter for a monitor-alert Automation and refreshes the 24h TTL; returns the new count. Inline because monitor-alert is the only consumer today. */
async function incrementAutomationFailure(args: {
projectId: string;
automationId: string;
}): Promise<number> {
if (!redis) return 0;
const key = automationFailureKey(args.projectId, args.automationId);
const results = await redis
.multi()
.incr(key)
.expire(key, automationFailureTtlSeconds)
.exec();
return Number(results?.[0]?.[1] ?? 0);
}
/** resetAutomationFailures clears the streak after a successful delivery. */
async function resetAutomationFailures(args: {
projectId: string;
automationId: string;
}): Promise<void> {
if (!redis) return;
await redis.del(automationFailureKey(args.projectId, args.automationId));
}
/** WebhookOutboundPayload is the validated HTTP body of a WebhookInput. */
type WebhookOutboundPayload = ReturnType<typeof buildWebhookOutboundPayload>;
/** buildWebhookOutboundPayload validates and returns the HTTP body for a WebhookInput, discriminating on payload.type. monitor-alert: the processor already built the unified envelope — pass through. prompt-version: wrap at dispatch time with id = executionId and timestamp = now. */
function buildWebhookOutboundPayload(input: WebhookInput) {
if (input.payload.type === "monitor-alert") {
const parsed = MonitorWebhookQueueEventSchema.safeParse(input.payload);
if (!parsed.success) {
throw new InternalServerError(
`Invalid monitor-alert payload: ${parsed.error.message}`,
);
}
return parsed.data;
}
if (input.payload.type === "project-notification") {
const parsed = ProjectNotificationWebhookQueueEventSchema.safeParse(
input.payload,
);
if (!parsed.success) {
throw new InternalServerError(
`Invalid project notification payload: ${parsed.error.message}`,
);
}
return parsed.data;
}
if (input.payload.type === "skill-version") {
return SkillWebhookOutboundSchema.parse({
id: input.executionId,
timestamp: new Date(),
apiVersion: "v1",
...input.payload,
});
}
const promptPayload = input.payload;
const parsed = PromptWebhookOutboundSchema.safeParse({
id: input.executionId,
timestamp: new Date(),
type: promptPayload.type,
apiVersion: "v1",
action: promptPayload.action,
prompt: promptPayload.prompt,
user: promptPayload.user
? { name: promptPayload.user.name, email: promptPayload.user.email }
: undefined,
});
if (!parsed.success) {
throw new InternalServerError(
`Invalid webhook payload: ${parsed.error.message}`,
);
}
return parsed.data;
}
// Handles both webhook and slack actions
export const webhookProcessor: Processor = async (
job: Job<TQueueJobTypes[QueueName.WebhookQueue]>,
) => {
try {
return await executeWebhook(job.data.payload);
} catch (error) {
logger.error("Error executing WebhookJob", error);
throw error;
}
};
// TODO: Webhook outgoing API versioning
export const executeWebhook = async (
input: WebhookInput,
options?: { skipValidation?: boolean },
) => {
const { projectId, automationId } = input;
try {
logger.debug(`Executing action for automation ${automationId}`);
const automation = await getAutomationById({
projectId,
automationId,
});
if (!automation) {
logger.warn(
`Automation ${automationId} not found for project ${projectId}. We ack the job and will not retry.`,
);
return;
}
if (
await isStaleMonitorAlert({
projectId,
automationId,
actionType: automation.action.type,
input,
})
) {
return;
}
// Route to appropriate handler based on action type
if (automation.action.type === "WEBHOOK") {
await executeWebhookAction({
input,
automation,
skipValidation: options?.skipValidation,
});
} else if (automation.action.type === "SLACK") {
await executeSlackAction({
input,
automation,
});
} else if (automation.action.type === "GITHUB_DISPATCH") {
await executeGitHubDispatchAction({
input,
automation,
skipValidation: options?.skipValidation,
});
} else if (automation.action.type === "PAGERDUTY") {
await executePagerDutyAction({ input, automation });
} else {
throw new InternalServerError(
`Unsupported action type: ${automation.action.type}`,
);
}
logger.debug(
`Action executed successfully for action ${automation.action.id}`,
);
} catch (error) {
logger.error("Error executing action", error);
throw error;
}
};
/** isStaleMonitorAlert prevents older queued alerts from overwriting newer published alerts by reporting which jobs to drop. */
async function isStaleMonitorAlert({
projectId,
automationId,
actionType,
input,
}: {
projectId: string;
automationId: string;
actionType: string;
input: WebhookInput;
}): Promise<boolean> {
if (input.payload.type !== "monitor-alert") return false;
const envelope = buildWebhookOutboundPayload(input);
if (envelope.type !== "monitor-alert") return false;
const alert = envelope.payload;
const reason = await getStaleMonitorAlertReason({ projectId, alert });
if (!reason) return false;
recordStaleMonitorAlertDrop({
projectId,
automationId,
actionType,
monitorId: alert.monitorId,
reason,
});
return true;
}
/** getStaleMonitorAlertReason says why an alert is stale, or undefined when it is current. */
async function getStaleMonitorAlertReason({
projectId,
alert,
}: {
projectId: string;
alert: MonitorAlert;
}): Promise<StaleMonitorAlertReason | undefined> {
const monitor = await prisma.monitor.findUnique({
where: { id: alert.monitorId },
select: { projectId: true, alertedAt: true },
});
if (!monitor || monitor.projectId !== projectId) return "monitor_deleted";
// The processor publishes before it saves alertedAt, so only a later alertedAt is newer.
if (
alert.alertedAt &&
monitor.alertedAt &&
monitor.alertedAt > alert.alertedAt
) {
return "newer_alert";
}
return undefined;
}
/** recordStaleMonitorAlertDrop logs a dropped stale alert and counts it by reason and action type. */
function recordStaleMonitorAlertDrop({
projectId,
automationId,
actionType,
monitorId,
reason,
}: {
projectId: string;
automationId: string;
actionType: string;
monitorId: string;
reason: StaleMonitorAlertReason;
}): void {
logger.info("Dropping stale monitor alert", {
projectId,
automationId,
monitorId,
reason,
});
recordIncrement(
"langfuse.queue.webhook_queue.stale_monitor_alerts_dropped",
1,
{
reason,
actionType,
},
);
}
/**
* Shared HTTP execution logic for webhook and GitHub dispatch actions
* Handles: fetch with timeout, retry logic, status validation, execution tracking
*/
async function executeHttpAction({
url,
payload,
headers,
projectId,
skipValidation,
automation,
executionId,
executionStart,
actionConfig,
additionalSensitiveHeaders,
payloadType,
whitelist,
addressAttemptTimeoutMs,
}: {
url: string;
payload: string;
headers: Record<string, string>;
projectId: string;
skipValidation?: boolean;
automation: AutomationDomain;
executionId: string;
executionStart: Date;
actionConfig: ActionDomainWithSecrets;
additionalSensitiveHeaders?: string[];
payloadType: WebhookInput["payload"]["type"];
whitelist: WebhookValidationWhitelist;
addressAttemptTimeoutMs?: number;
}): Promise<{ httpStatus: number; responseBody: string }> {
// monitor-alert does not write AutomationExecution rows; auto-disable rides a Redis counter instead.
const isMonitorAlert = payloadType === "monitor-alert";
let httpStatus: number | undefined;
let responseBody: string | undefined;
try {
// Execute HTTP request with retries
await backOff(
async () => {
logger.info(
`Sending HTTP request to ${urlForLog(url)} for action ${automation.action.id}`,
);
// Create AbortController for timeout
const abortController = new AbortController();
const timeoutId = setTimeout(() => {
abortController.abort();
}, env.LANGFUSE_WEBHOOK_TIMEOUT_MS);
try {
// Skip validation when flag is set (for tests with MSW mocking)
if (!skipValidation) {
await validateWebhookURL(url, whitelist);
}
const redirectOptions = skipValidation
? {
maxRedirects: env.LANGFUSE_WEBHOOK_MAX_REDIRECTS,
skipValidation: true as const,
additionalSensitiveHeaders,
}
: {
maxRedirects: env.LANGFUSE_WEBHOOK_MAX_REDIRECTS,
redirectValidation: {
validateUrl: validateWebhookURL,
whitelist,
logContext: WEBHOOK_URL_VALIDATION_LOG_CONTEXT,
addressAttemptTimeoutMs,
},
additionalSensitiveHeaders,
};
const redirectResult = await fetchWithSecureRedirects(
url,
{
method: "POST",
body: payload,
headers,
signal: abortController.signal,
},
redirectOptions,
);
const res = redirectResult.response;
// Log redirect chain if any redirects occurred (security audit trail)
if (redirectResult.redirectChain.length > 0) {
logger.info("Webhook followed redirects", {
actionId: automation.action.id,
initialUrl: urlForLog(url),
finalUrl: urlForLog(redirectResult.finalUrl),
redirectCount: redirectResult.redirectChain.length,
redirectChain: redirectResult.redirectChain.map(urlForLog),
});
}
httpStatus = res.status;
responseBody = await res.text();
if (!res.ok) {
logger.warn(
`Webhook does not return 2xx status: failed with status ${res.status} for url ${urlForLog(url)} and project ${projectId}. Body: ${responseBody}`,
);
throw new Error(
`Webhook does not return 2xx status: failed with status ${res.status} for url ${urlForLog(url)} and project ${projectId}`,
);
}
} catch (error) {
if (error instanceof Error && error.name === "AbortError") {
logger.warn(
`Webhook timeout after ${env.LANGFUSE_WEBHOOK_TIMEOUT_MS}ms for url ${urlForLog(url)} and project ${projectId}`,
);
throw new Error(
`Webhook timeout after ${env.LANGFUSE_WEBHOOK_TIMEOUT_MS}ms for url ${urlForLog(url)} and project ${projectId}`,
);
}
throw error;
} finally {
clearTimeout(timeoutId);
}
},
{
numOfAttempts: 4,
},
);
// Update execution status on success
if (isMonitorAlert) {
try {
await resetAutomationFailures({
projectId,
automationId: automation.id,
});
} catch (resetError) {
logger.warn(
`Failed to reset automation failure counter after successful delivery for automation ${automation.id} in project ${projectId}`,
resetError,
);
}
} else {
await prisma.automationExecution.update({
where: {
projectId,
triggerId: automation.trigger.id,
actionId: automation.action.id,
id: executionId,
},
data: {
status: ActionExecutionStatus.COMPLETED,
startedAt: executionStart,
finishedAt: new Date(),
},
});
}
return { httpStatus: httpStatus!, responseBody: responseBody! };
} catch (error) {
logger.error("Error executing HTTP action", error);
// Handle action failure with retry logic and trigger disabling
const shouldRetryJob =
error instanceof LangfuseNotFoundError ||
error instanceof InternalServerError;
if (shouldRetryJob) {
logger.warn(`Retrying BullMQ for action ${automation.action.id}`);
throw error; // Trigger BullMQ retry
}
if (!actionConfig) {
logger.error("Action config not found for failure handling");
throw error;
}
if (isMonitorAlert) {
// monitor-alert: Redis counter compensates for the missing
// AutomationExecution streak. Disable on the 5th consecutive failure.
const count = await incrementAutomationFailure({
projectId,
automationId: automation.id,
});
if (count >= automationFailureThreshold) {
await prisma.trigger.update({
where: { id: automation.trigger.id, projectId },
data: { status: JobConfigState.INACTIVE },
});
try {
await resetAutomationFailures({
projectId,
automationId: automation.id,
});
} catch (resetError) {
logger.warn(
`Failed to reset automation failure counter after auto-disable for automation ${automation.id} in project ${projectId}`,
resetError,
);
}
logger.warn(
`Automation ${automation.trigger.id} disabled after ${count} consecutive failures in project ${projectId} (monitor-alert)`,
);
}
return { httpStatus: httpStatus || 0, responseBody: responseBody || "" };
}
// project-notification records the ERROR row for visibility but never
// auto-disables; prompt-version / github keep the consecutive-failure
// disable behavior below.
const disablesOnFailure = payloadType !== "project-notification";
// Update execution status and check if we should disable trigger
await prisma.$transaction(async (tx) => {
// Update execution status
await tx.automationExecution.update({
where: {
id: executionId,
projectId,
triggerId: automation.trigger.id,
actionId: automation.action.id,
},
data: {
status: ActionExecutionStatus.ERROR,
startedAt: executionStart,
finishedAt: new Date(),
error: error instanceof Error ? error.message : "Unknown error",
output: httpStatus
? {
httpStatus,
responseBody: responseBody?.substring(0, 1000),
}
: undefined,
},
});
if (!disablesOnFailure) {
return;
}
// Check consecutive failures from execution history
const consecutiveFailures = await getConsecutiveAutomationFailures({
automationId: automation.id,
projectId,
});
logger.info(
`Consecutive failures: ${consecutiveFailures} for trigger ${automation.trigger.id} in project ${projectId}`,
);
// Check if trigger should be disabled (this is the 5th failure, looking for 4 in the past.)
if (consecutiveFailures >= 4) {
// Update trigger to inactive status
await tx.trigger.update({
where: { id: automation.trigger.id, projectId },
data: { status: JobConfigState.INACTIVE },
});
// Update action config to store the failing execution ID
// Type guard to ensure we only update webhook or GitHub dispatch configs
if (
isWebhookAction(actionConfig) ||
isGitHubDispatchAction(actionConfig)
) {
await setActionLastFailingExecutionId({
tx,
actionId: automation.action.id,
projectId,
executionId,
});
}
logger.warn(
`Automation ${automation.trigger.id} disabled after ${consecutiveFailures} consecutive failures in project ${projectId}`,
);
}
});
logger.debug(
`HTTP action failed for action ${automation.action.id} in project ${projectId}`,
);
// Error has been handled - don't rethrow
// Return empty response to indicate failure was handled
return { httpStatus: httpStatus || 0, responseBody: responseBody || "" };
}
}
/**
* Execute webhook action with HTTP request and signature validation
*/
async function executeWebhookAction({
input,
automation,
skipValidation,
}: {
input: WebhookInput;
automation: Awaited<ReturnType<typeof getAutomationById>>;
skipValidation?: boolean;
}) {
if (!automation) return;
const { projectId, executionId } = input;
const executionStart = new Date();
const actionConfig = await getActionByIdWithSecrets({
projectId,
actionId: automation.action.id,
});
if (!actionConfig) {
throw new InternalServerError("Action config not found");
}
if (!isWebhookAction(actionConfig)) {
throw new InternalServerError(
"Action config is not a valid webhook configuration",
);
}
const webhookConfig = actionConfig.config;
const envelope = buildWebhookOutboundPayload(input);
let webhookPayload: string;
if (input.payload.type === "prompt-version") {
// Prompt-version contract: prompt key always last.
const { prompt, user, ...otherFields } = envelope as Extract<
typeof envelope,
{ type: "prompt-version" }
>;
webhookPayload = JSON.stringify({
...otherFields,
...(user ? { user } : {}),
prompt,
});
} else {
// monitor-alert: post the envelope verbatim.
webhookPayload = JSON.stringify(envelope);
}
let requestHeaders: Record<string, string>;
let additionalSensitiveHeaders: string[];
try {
const builtRequestHeaders = buildWebhookRequestHeaders({
customHeaders: webhookConfig.requestHeaders,
body: webhookPayload,
signingSecret: decrypt(webhookConfig.secretKey),
});
requestHeaders = builtRequestHeaders.headers;
additionalSensitiveHeaders = builtRequestHeaders.sensitiveHeaderNames;
} catch (error) {
logger.error(
"Failed to decrypt webhook secret or generate signature",
error,
);
throw new InternalServerError("Failed to generate webhook signature");
}
// Execute HTTP request using shared function
await executeHttpAction({
url: webhookConfig.url,
payload: webhookPayload,
headers: requestHeaders,
projectId,
skipValidation,
automation,
executionId,
executionStart,
actionConfig,
additionalSensitiveHeaders,
payloadType: input.payload.type,
whitelist: webhookWhitelistFromEnv(),
addressAttemptTimeoutMs: WEBHOOK_ADDRESS_ATTEMPT_TIMEOUT_MS,
});
}
function buildSkillDispatchPayload(
validated: Extract<
ReturnType<typeof buildWebhookOutboundPayload>,
{ type: "skill-version" }
>,
eventType: string,
): string {
let githubPayload = JSON.stringify({
event_type: eventType,
client_payload: validated,
});
if (
Buffer.byteLength(githubPayload, "utf8") >=
GITHUB_REPOSITORY_DISPATCH_MAX_PAYLOAD_BYTES
) {
githubPayload = JSON.stringify({
event_type: eventType,
client_payload: {
id: validated.id,
timestamp: validated.timestamp,
type: validated.type,
apiVersion: validated.apiVersion,
action: validated.action,
skill: {
id: validated.skill.id,
name: validated.skill.name,
version: validated.skill.version,
projectId: validated.skill.projectId,
},
truncation: {
payloadTruncated: true,
truncatedFields: [
"skill.files",
"skill.frontmatter",
"skill.description",
"skill.tags",
"skill.labels",
"skill.commitMessage",
"skill.createdBy",
"skill.createdAt",
"skill.updatedAt",
"user",
],
},
},
});
}
if (
Buffer.byteLength(githubPayload, "utf8") >=
GITHUB_REPOSITORY_DISPATCH_MAX_PAYLOAD_BYTES
) {
throw new InternalServerError(
"Skill repository dispatch payload exceeds GitHub's size limit",
);
}
return githubPayload;
}
/**
* Execute GitHub Dispatch action with repository dispatch API
*/
async function executeGitHubDispatchAction({
input,
automation,
skipValidation,
}: {
input: WebhookInput;
automation: Awaited<ReturnType<typeof getAutomationById>>;
skipValidation?: boolean;
}) {
if (!automation) return;
const { projectId, executionId } = input;
const executionStart = new Date();
const actionConfig = await getActionByIdWithSecrets({
projectId,
actionId: automation.action.id,
});
if (!actionConfig) {
throw new InternalServerError("Action config not found");
}
if (!isGitHubDispatchAction(actionConfig)) {
throw new InternalServerError(
"Action config is not a valid GitHub dispatch configuration",
);
}
const githubConfig = actionConfig.config;
const eventType = githubConfig.eventType;
const envelope = buildWebhookOutboundPayload(input);
let githubPayload: string;
if (input.payload.type === "prompt-version") {
const { prompt, user, ...otherFields } = envelope as Extract<
typeof envelope,
{ type: "prompt-version" }
>;
const fullGithubPayload = JSON.stringify({
event_type: eventType,
client_payload: {
...otherFields,
...(user ? { user } : {}),
prompt,
},
});
githubPayload =
Buffer.byteLength(fullGithubPayload, "utf8") <
GITHUB_REPOSITORY_DISPATCH_MAX_PAYLOAD_BYTES
? fullGithubPayload
: JSON.stringify({
event_type: eventType,
client_payload: {
...otherFields,
...(user ? { user } : {}),
truncation: {
payloadTruncated: true,
truncatedFields: GITHUB_REPOSITORY_DISPATCH_TRUNCATED_FIELDS,
},
prompt: {
...prompt,
prompt: GITHUB_REPOSITORY_DISPATCH_TRUNCATION_MARKER,
config: {},
},
},
});
} else if (envelope.type === "skill-version") {
githubPayload = buildSkillDispatchPayload(envelope, eventType);
} else {
// monitor-alert: post the envelope as client_payload.
const monitorEnvelope = envelope as Extract<
typeof envelope,
{ type: "monitor-alert" }
>;
const fullGithubPayload = JSON.stringify({
event_type: eventType,
client_payload: monitorEnvelope,
});
githubPayload =
Buffer.byteLength(fullGithubPayload, "utf8") <
GITHUB_REPOSITORY_DISPATCH_MAX_PAYLOAD_BYTES
? fullGithubPayload
: JSON.stringify({
event_type: eventType,
client_payload: {
...monitorEnvelope,
payload: {
...monitorEnvelope.payload,
filters: [],
message: {
...monitorEnvelope.payload.message,
body: GITHUB_REPOSITORY_DISPATCH_TRUNCATION_MARKER,
},
},
truncation: {
payloadTruncated: true,
truncatedFields: ["payload.filters", "payload.message.body"],
},
},
});
}
// Prepare headers with GitHub token
const requestHeaders: Record<string, string> = {};
// Add default headers
for (const [key, value] of Object.entries(WebhookDefaultHeaders)) {
requestHeaders[key] = value;
}
try {
const decryptedToken = decrypt(githubConfig.githubToken);
// Add GitHub token as Bearer auth
requestHeaders["Authorization"] = `Bearer ${decryptedToken}`;
// Add signature for optional verification (using GitHub token as secret)
const signature = createSignatureHeader(githubPayload, decryptedToken);
requestHeaders["x-langfuse-signature"] = signature;
} catch (error) {
logger.error("Failed to decrypt GitHub token or generate signature", error);
throw new InternalServerError("Failed to generate GitHub authentication");
}
// Execute HTTP request using shared function
await executeHttpAction({
url: githubConfig.url,
payload: githubPayload,
headers: requestHeaders,
projectId,
skipValidation,
automation,
executionId,
executionStart,
actionConfig,
payloadType: input.payload.type,
whitelist: whitelistFromEnv(),
});
}
/**
* Execute Slack action with message sending via SlackService
*/
async function executeSlackAction({
input,
automation,
}: {
input: WebhookInput;
automation: Awaited<ReturnType<typeof getAutomationById>>;
}) {
if (!automation) return;
const { projectId, executionId } = input;
const executionStart = new Date();
const actionConfig = await getActionById({
projectId,
actionId: automation.action.id,
});
if (!actionConfig) {
throw new InternalServerError("Action config not found");
}
if (!isSlackActionConfig(actionConfig.config)) {
throw new InternalServerError(
"Action config is not a valid Slack configuration",
);
}
const slackConfig = actionConfig.config;
// TODO: unify automation execution log and failure handling across all message types (not scalable yet)
// TODO: implement a strategy/policy + registry pattern for different webhook providers.
if (input.payload.type === "monitor-alert") {
try {
await sendSlackMonitorAlert({
payload: input.payload,
projectId,
channelId: slackConfig.channelId,
automationId: automation.id,
});
} catch (error) {
logger.error("Error executing Slack action", error);
await monitorAlertDeliveryFailure({
projectId,
automation,
channel: "slack",
});
}
return;
}
if (input.payload.type === "project-notification") {
// Project notifications resolve their AutomationExecution row (so the
// executions table stays accurate) but never auto-disable — a persistently
// failing channel keeps recording ERROR rows and stays ACTIVE.
try {
await sendSlackProjectNotification({
payload: input.payload,
projectId,
channelId: slackConfig.channelId,
});
await prisma.automationExecution.update({
where: {
projectId,
triggerId: automation.trigger.id,
actionId: automation.action.id,
id: executionId,
},
data: {
status: ActionExecutionStatus.COMPLETED,
startedAt: executionStart,
finishedAt: new Date(),
},
});
} catch (error) {
logger.error("Error executing Slack action", error);
await prisma.automationExecution.update({
where: {
id: executionId,
projectId,
triggerId: automation.trigger.id,
actionId: automation.action.id,
},
data: {
status: ActionExecutionStatus.ERROR,
startedAt: executionStart,
finishedAt: new Date(),
error: error instanceof Error ? error.message : "Unknown error",
},
});
}
return;
}
try {
// Build message blocks using predefined formats or custom template
let blocks: any[] = [];
let attachments:
| { color: string; fallback?: string; blocks?: any[] }[]
| undefined;
// TODO: Custom templates not supported via the UI yet
if (slackConfig.messageTemplate) {
try {
blocks = JSON.parse(slackConfig.messageTemplate);
logger.debug(
`Using custom message template for action ${automation.action.id}`,
);
} catch (error) {
logger.warn(
`Invalid Slack messageTemplate JSON for action ${automation.action.id}. Using default format`,
{ error: error instanceof Error ? error.message : "Unknown error" },
);
}
}
// Use predefined message format if no custom template or template failed
if (blocks.length === 0) {
const message = SlackMessageBuilder.buildMessage(input.payload);
blocks = message.blocks;
attachments = message.attachments;
logger.debug(
`Using predefined message format for action ${automation.action.id}`,
);
}
// Get Slack WebClient for project via centralized SlackService
const client =
await SlackService.getInstance().getWebClientForProject(projectId);
// Send message
const sendResult = await SlackService.getInstance().sendMessage({
client,
channelId: slackConfig.channelId,
blocks,
attachments,
text: "Langfuse Notification",
});
// Update execution status to completed
await prisma.automationExecution.update({
where: {
projectId,
triggerId: automation.trigger.id,
actionId: automation.action.id,
id: executionId,
},
data: {
status: ActionExecutionStatus.COMPLETED,
startedAt: executionStart,
finishedAt: new Date(),
output: {
channel: sendResult.channel,
messageTs: sendResult.messageTs,
},
},
});
} catch (error) {
logger.error("Error executing Slack action", error);
// Get action config for updating in case of failure
const failureActionConfig = await getActionByIdWithSecrets({
projectId,
actionId: automation.action.id,
});
if (!failureActionConfig) {
logger.error("Action config not found for failure handling");
return;
}
// Update execution status and disable trigger
await prisma.$transaction(async (tx) => {
// Update execution status
await tx.automationExecution.update({
where: {
id: executionId,
projectId,
triggerId: automation.trigger.id,
actionId: automation.action.id,
},
data: {
status: ActionExecutionStatus.ERROR,
startedAt: executionStart,
finishedAt: new Date(),
error: error instanceof Error ? error.message : "Unknown error",
},
});
// Update trigger to inactive status
await tx.trigger.update({
where: { id: automation.trigger.id, projectId },
data: { status: JobConfigState.INACTIVE },
});
// Update action config to store the failing execution ID
await setActionLastFailingExecutionId({
tx,
actionId: automation.action.id,
projectId,
executionId,
});
logger.warn(
`Automation ${automation.trigger.id} disabled after 1 failure in project ${projectId}`,
);
});
logger.debug(
`Slack action failed for action ${automation.action.id} in project ${projectId}`,
);
}
}
/**
* Execute PagerDuty action. Events go only to the fixed PagerDuty hosts of the
* routing key's region. 429, 5xx and network errors are retried in-process
* like HTTP actions; a failure that remains counts toward trigger auto-disable.
*/
async function executePagerDutyAction({
input,
automation,
}: {
input: WebhookInput;
automation: NonNullable<Awaited<ReturnType<typeof getAutomationById>>>;
}) {
const { projectId, executionId } = input;
const executionStart = new Date();
const actionConfig = await getActionByIdWithSecrets({
projectId,
actionId: automation.action.id,
});
if (!actionConfig || !isPagerDutyActionConfig(actionConfig.config)) {
throw new InternalServerError("PagerDuty action config not found");
}
const config = actionConfig.config;
try {
const envelope = buildWebhookOutboundPayload(input);
const event = await buildPagerDutyDelivery({ input, envelope, config });
if (!event) {
logger.info(
`No PagerDuty event for ${input.payload.type} of automation ${automation.id} in project ${projectId}`,
);
return;
}
await sendPagerDutyEvent(event, config.region);
} catch (error) {
logger.error("Error executing PagerDuty action", error);
await recordPagerDutyFailure({ input, automation, executionStart, error });
return;
}
if (input.payload.type === "monitor-alert") {
await resetAutomationFailures({ projectId, automationId: automation.id });
return;
}
await prisma.automationExecution.update({
where: {
projectId,
triggerId: automation.trigger.id,
actionId: automation.action.id,
id: executionId,
},
data: {
status: ActionExecutionStatus.COMPLETED,
startedAt: executionStart,
finishedAt: new Date(),
},
});
}
/** buildPagerDutyDelivery decrypts the routing key and maps the job payload to its PagerDuty event. */
async function buildPagerDutyDelivery({
input,
envelope,
config,
}: {
input: WebhookInput;
envelope: WebhookOutboundPayload;
config: PagerDutyActionConfigWithSecrets;
}): Promise<PagerDutyEvent | undefined> {
const project = await prisma.project.findUniqueOrThrow({
where: { id: input.projectId },
select: { name: true },
});
return buildPagerDutyEvent({
envelope,
projectId: input.projectId,
projectName: project.name,
routingKey: decrypt(config.routingKey),
});
}
/** sendPagerDutyEvent posts the event to the alert or change endpoint of the key's region. */
async function sendPagerDutyEvent(
event: PagerDutyEvent,
region: PagerDutyRegion,
) {
if (event.kind === "change") {
await pagerDutyClient.change({ region, event: event.event });
return;
}
await pagerDutyClient.alert({ region, event: event.event });
}
/** recordPagerDutyFailure records a non-retryable delivery failure and auto-disables the trigger like the other action types. */
async function recordPagerDutyFailure({
input,
automation,
executionStart,
error,
}: {
input: WebhookInput;
automation: NonNullable<Awaited<ReturnType<typeof getAutomationById>>>;
executionStart: Date;
error: unknown;
}) {
const { projectId, executionId } = input;
if (input.payload.type === "monitor-alert") {
await monitorAlertDeliveryFailure({
projectId,
automation,
channel: "pagerduty",
});
return;
}
await prisma.$transaction(async (tx) => {
await tx.automationExecution.update({
where: {
id: executionId,
projectId,
triggerId: automation.trigger.id,
actionId: automation.action.id,
},
data: {
status: ActionExecutionStatus.ERROR,
startedAt: executionStart,
finishedAt: new Date(),
error: error instanceof Error ? error.message : "Unknown error",
output: isBaseError(error) ? { httpStatus: error.httpCode } : undefined,
},
});
// Project notifications record the error but never auto-disable.
if (input.payload.type === "project-notification") return;
const consecutiveFailures = await getConsecutiveAutomationFailures({
automationId: automation.id,
projectId,
});
if (consecutiveFailures >= automationFailureThreshold - 1) {
await tx.trigger.update({
where: { id: automation.trigger.id, projectId },
data: { status: JobConfigState.INACTIVE },
});
logger.warn(
`Automation ${automation.trigger.id} disabled after ${consecutiveFailures} consecutive failures in project ${projectId} (pagerduty)`,
);
}
});
}
/** sendSlackMonitorAlert builds and posts a monitor-alert to Slack, then clears the failure streak. */
async function sendSlackMonitorAlert({
payload,
projectId,
channelId,
automationId,
}: {
payload: WebhookInput["payload"];
projectId: string;
channelId: string;
automationId: string;
}) {
const parsed = MonitorWebhookQueueEventSchema.safeParse(payload);
if (!parsed.success) {
throw new InternalServerError(
`Invalid monitor-alert payload: ${parsed.error.message}`,
);
}
const message = buildMonitorAlertSlackMessage(parsed.data.payload);
const client =
await SlackService.getInstance().getWebClientForProject(projectId);
await SlackService.getInstance().sendMessage({
client,
channelId,
...message,
});
await resetAutomationFailures({ projectId, automationId });
}
/** sendSlackProjectNotification builds and posts a project notification to Slack. */
async function sendSlackProjectNotification({
payload,
projectId,
channelId,
}: {
payload: WebhookInput["payload"];
projectId: string;
channelId: string;
}) {
const parsed = ProjectNotificationWebhookQueueEventSchema.safeParse(payload);
if (!parsed.success) {
throw new InternalServerError(
`Invalid project notification payload: ${parsed.error.message}`,
);
}
const message = buildProjectNotificationSlackMessage(parsed.data.event);
const client =
await SlackService.getInstance().getWebClientForProject(projectId);
await SlackService.getInstance().sendMessage({
client,
channelId,
...message,
});
}
/** monitorAlertDeliveryFailure tracks a failed monitor-alert delivery and auto-disables the trigger past the failure threshold. */
async function monitorAlertDeliveryFailure({
projectId,
automation,
channel,
}: {
projectId: string;
automation: NonNullable<Awaited<ReturnType<typeof getAutomationById>>>;
channel: "slack" | "pagerduty";
}) {
const count = await incrementAutomationFailure({
projectId,
automationId: automation.id,
});
if (count >= automationFailureThreshold) {
await prisma.trigger.update({
where: { id: automation.trigger.id, projectId },
data: { status: JobConfigState.INACTIVE },
});
await resetAutomationFailures({
projectId,
automationId: automation.id,
});
logger.warn(
`Automation ${automation.trigger.id} disabled after ${count} consecutive failures in project ${projectId} (monitor-alert/${channel})`,
);
}
}
const setActionLastFailingExecutionId = async ({
tx,
actionId,
projectId,
executionId,
}: {
tx: Prisma.TransactionClient;
actionId: string;
projectId: string;
executionId: string;
}) => {
// The execution config may contain decrypted headers, so patch only this JSON
// key and leave the stored encrypted config untouched.
await tx.$executeRaw`
UPDATE actions
SET
config = jsonb_set(
config,
'{lastFailingExecutionId}',
to_jsonb(${executionId}::text),
true
),
updated_at = NOW()
WHERE id = ${actionId}
AND project_id = ${projectId}
`;
};
/** StaleMonitorAlertReason is why a queued monitor alert was dropped. */
type StaleMonitorAlertReason = "monitor_deleted" | "newer_alert";