🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(pagerduty): send alerts, project notifications and prompt changes to PagerDuty, with the routing key on the automation (#19411)
Mark Salpeter7adf573f8e19
diff --git a/packages/shared/package.json b/packages/shared/package.jsonindex 2e628a22b3..1cd1a027f6 100644--- a/packages/shared/package.json+++ b/packages/shared/package.json@@ -135,6 +135,14 @@"import": "./dist/src/features/monitors/server.js","require": "./dist/src/features/monitors/server.js"},+ "./pagerduty": {+ "import": "./dist/src/features/pagerduty/index.js",+ "require": "./dist/src/features/pagerduty/index.js"+ },+ "./pagerduty/server": {+ "import": "./dist/src/features/pagerduty/server.js",+ "require": "./dist/src/features/pagerduty/server.js"+ },"./in-app-agent": {"import": "./dist/src/in-app-agent/index.js","require": "./dist/src/in-app-agent/index.js"diff --git a/packages/shared/prisma/migrations/20261026120100_add_pagerduty_action_type/migration.sql b/packages/shared/prisma/migrations/20261026120100_add_pagerduty_action_type/migration.sqlnew file mode 100644index 0000000000..32dfc5cf54--- /dev/null+++ b/packages/shared/prisma/migrations/20261026120100_add_pagerduty_action_type/migration.sql@@ -0,0 +1,6 @@+-- Migration: Add PagerDuty Action Type+-- This migration adds support for PagerDuty actions by:+-- 1. Adding PAGERDUTY to the ActionType enum++-- AlterEnum+ALTER TYPE "ActionType" ADD VALUE 'PAGERDUTY';diff --git a/packages/shared/prisma/schema.prisma b/packages/shared/prisma/schema.prismaindex 80c9115e96..fa5f514b9a 100644--- a/packages/shared/prisma/schema.prisma+++ b/packages/shared/prisma/schema.prisma@@ -2032,6 +2032,7 @@ enum ActionType {WEBHOOKSLACKGITHUB_DISPATCH+ PAGERDUTY// More action types can be added as needed}diff --git a/packages/shared/src/domain/automations.ts b/packages/shared/src/domain/automations.tsindex 3bbf81e6b8..9b941903cc 100644--- a/packages/shared/src/domain/automations.ts+++ b/packages/shared/src/domain/automations.ts@@ -1,6 +1,10 @@import { Action, Trigger } from "@prisma/client";import { FilterState } from "../types";import { z } from "zod";+import {+ PagerDutyRegionSchema,+ PagerDutyRoutingKeySchema,+} from "../features/pagerduty/types";export enum TriggerEventSource {Prompt = "prompt",@@ -60,7 +64,12 @@ export type ActionDomainWithSecrets = Omit<Action, "config"> & {config: ActionConfigWithSecrets;};-export const ActionTypeSchema = z.enum(["WEBHOOK", "SLACK", "GITHUB_DISPATCH"]);+export const ActionTypeSchema = z.enum([+ "WEBHOOK",+ "SLACK",+ "GITHUB_DISPATCH",+ "PAGERDUTY",+]);export const AvailableWebhookApiSchema = z.partialRecord(z.enum(["prompt", "monitor", "project-notification"]),@@ -144,22 +153,51 @@ export type GitHubDispatchActionConfigWithSecrets = z.infer<typeof GitHubDispatchActionConfigSchema>;+/** PagerDutyActionConfigSchema is a stored PagerDuty action; `routingKey` is encrypted. */+export const PagerDutyActionConfigSchema = z.object({+ type: z.literal("PAGERDUTY"),+ routingKey: z.string(),+ displayRoutingKey: z.string(),+ region: PagerDutyRegionSchema,+});++export const SafePagerDutyActionConfigSchema = PagerDutyActionConfigSchema.omit(+ { routingKey: true },+);++/** PagerDutyActionCreateSchema is the PagerDuty action input; an update without `routingKey` keeps the stored key. */+export const PagerDutyActionCreateSchema = z.object({+ type: z.literal("PAGERDUTY"),+ routingKey: PagerDutyRoutingKeySchema.optional(),+ region: PagerDutyRegionSchema,+});++export type PagerDutyActionConfigWithSecrets = z.infer<+ typeof PagerDutyActionConfigSchema+>;+export type SafePagerDutyActionConfig = z.infer<+ typeof SafePagerDutyActionConfigSchema+>;+export const ActionConfigSchema = z.discriminatedUnion("type", [WebhookActionConfigSchema,SlackActionConfigSchema,GitHubDispatchActionConfigSchema,+ PagerDutyActionConfigSchema,]);export const ActionCreateSchema = z.discriminatedUnion("type", [WebhookActionCreateSchema,SlackActionConfigSchema,GitHubDispatchActionCreateSchema,+ PagerDutyActionCreateSchema,]);export const SafeActionConfigSchema = z.discriminatedUnion("type", [SafeWebhookActionConfigSchema,SlackActionConfigSchema,SafeGitHubDispatchActionConfigSchema,+ SafePagerDutyActionConfigSchema,]);export type ActionTypes = z.infer<typeof ActionTypeSchema>;@@ -278,3 +316,10 @@ export function convertToSafeGitHubDispatchConfig(lastFailingExecutionId: config.lastFailingExecutionId,};}++/** isPagerDutyActionConfig checks for a valid PagerDuty action configuration. */+export function isPagerDutyActionConfig(+ config: unknown,+): config is PagerDutyActionConfigWithSecrets {+ return PagerDutyActionConfigSchema.safeParse(config).success;+}diff --git a/packages/shared/src/errors/BadGatewayError.ts b/packages/shared/src/errors/BadGatewayError.tsnew file mode 100644index 0000000000..72edbad293--- /dev/null+++ b/packages/shared/src/errors/BadGatewayError.ts@@ -0,0 +1,8 @@+import { BaseError } from "./BaseError";++/** BadGatewayError reports an invalid response from an upstream service. */+export class BadGatewayError extends BaseError {+ constructor(description = "Bad Gateway") {+ super("BadGatewayError", 502, description, true);+ }+}diff --git a/packages/shared/src/errors/GatewayTimeoutError.ts b/packages/shared/src/errors/GatewayTimeoutError.tsnew file mode 100644index 0000000000..4a465d174d--- /dev/null+++ b/packages/shared/src/errors/GatewayTimeoutError.ts@@ -0,0 +1,8 @@+import { BaseError } from "./BaseError";++/** GatewayTimeoutError reports that an upstream service did not respond in time. */+export class GatewayTimeoutError extends BaseError {+ constructor(description = "Gateway Timeout") {+ super("GatewayTimeoutError", 504, description, true);+ }+}diff --git a/packages/shared/src/errors/TooManyRequestsError.ts b/packages/shared/src/errors/TooManyRequestsError.tsnew file mode 100644index 0000000000..9389bba5ff--- /dev/null+++ b/packages/shared/src/errors/TooManyRequestsError.ts@@ -0,0 +1,8 @@+import { BaseError } from "./BaseError";++/** TooManyRequestsError reports that a rate limit rejected the request. */+export class TooManyRequestsError extends BaseError {+ constructor(description = "Too Many Requests") {+ super("TooManyRequestsError", 429, description, true);+ }+}diff --git a/packages/shared/src/errors/index.ts b/packages/shared/src/errors/index.tsindex 1bf12c806d..76171be3a8 100644--- a/packages/shared/src/errors/index.ts+++ b/packages/shared/src/errors/index.ts@@ -10,4 +10,8 @@ export { LangfuseConflictError } from "./ConflictError";export { ServiceUnavailableError } from "./ServiceUnavailableError";export { NotImplementedError } from "./NotImplementedError";export { PayloadTooLargeError } from "./PayloadTooLargeError";+export { TooManyRequestsError } from "./TooManyRequestsError";+export { BadGatewayError } from "./BadGatewayError";+export { GatewayTimeoutError } from "./GatewayTimeoutError";+export { toBaseError } from "./toBaseError";export * from "./errorMessages";diff --git a/packages/shared/src/errors/toBaseError.test.ts b/packages/shared/src/errors/toBaseError.test.tsnew file mode 100644index 0000000000..e1f3915da7--- /dev/null+++ b/packages/shared/src/errors/toBaseError.test.ts@@ -0,0 +1,36 @@+import { describe, expect, expectTypeOf, it } from "vitest";++import { BaseError } from "./BaseError";+import { TooManyRequestsError } from "./TooManyRequestsError";+import { toBaseError } from "./toBaseError";++describe("toBaseError", () => {+ it.each([+ [404, "LangfuseNotFoundError"],+ [429, "TooManyRequestsError"],+ [502, "BadGatewayError"],+ [504, "GatewayTimeoutError"],+ [418, "HttpError"],+ [599, "HttpError"],+ ])("maps %i to %s", (status, name) => {+ const error = toBaseError(status, "upstream said no");++ expect(error).toBeInstanceOf(BaseError);+ expect(error).toMatchObject({+ name,+ httpCode: status,+ message: "upstream said no",+ });+ });++ it("uses the error's default message when none is given", () => {+ expect(toBaseError(429).message).toBe("Too Many Requests");+ expect(toBaseError(418).message).toBe("HTTP 418");+ });++ it("types a literal status as its mapped error", () => {+ expectTypeOf(toBaseError(429)).toEqualTypeOf<TooManyRequestsError>();+ expectTypeOf(toBaseError(418)).toEqualTypeOf<BaseError>();+ expectTypeOf(toBaseError(Number("429"))).toEqualTypeOf<BaseError>();+ });+});diff --git a/packages/shared/src/errors/toBaseError.ts b/packages/shared/src/errors/toBaseError.tsnew file mode 100644index 0000000000..14a730c155--- /dev/null+++ b/packages/shared/src/errors/toBaseError.ts@@ -0,0 +1,58 @@+import { BadGatewayError } from "./BadGatewayError";+import { BaseError } from "./BaseError";+import { LangfuseConflictError } from "./ConflictError";+import { ForbiddenError } from "./ForbiddenError";+import { GatewayTimeoutError } from "./GatewayTimeoutError";+import { InternalServerError } from "./InternalServerError";+import { InvalidRequestError } from "./InvalidRequestError";+import { MethodNotAllowedError } from "./MethodNotAllowedError";+import { LangfuseNotFoundError } from "./NotFoundError";+import { NotImplementedError } from "./NotImplementedError";+import { ServiceUnavailableError } from "./ServiceUnavailableError";+import { TooManyRequestsError } from "./TooManyRequestsError";+import { UnauthorizedError } from "./UnauthorizedError";++const errorsByStatus = {+ 400: InvalidRequestError,+ 401: UnauthorizedError,+ 403: ForbiddenError,+ 404: LangfuseNotFoundError,+ 405: MethodNotAllowedError,+ 409: LangfuseConflictError,+ 429: TooManyRequestsError,+ 500: InternalServerError,+ 501: NotImplementedError,+ 502: BadGatewayError,+ 503: ServiceUnavailableError,+ 504: GatewayTimeoutError,+} as const;++/** toBaseError maps an HTTP status to its base error, or to a plain BaseError with that status when none is mapped. */+export function toBaseError<Status extends number>(+ status: Status,+ message?: string,+): BaseErrorForStatus<Status> {+ const ErrorClass: BaseErrorClass | undefined =+ errorsByStatus[status as keyof ErrorsByStatus];+ if (!ErrorClass) {+ return new BaseError(+ "HttpError",+ status,+ message ?? `HTTP ${status}`,+ true,+ ) as BaseErrorForStatus<Status>;+ }+ return new ErrorClass(message) as BaseErrorForStatus<Status>;+}++/** ErrorsByStatus is the status-to-class table toBaseError reads. */+type ErrorsByStatus = typeof errorsByStatus;++/** BaseErrorForStatus is the error class mapped to a literal status, and BaseError for any other status. */+type BaseErrorForStatus<Status extends number> =+ Status extends keyof ErrorsByStatus+ ? InstanceType<ErrorsByStatus[Status]>+ : BaseError;++/** BaseErrorClass is a base error constructor taking an optional message. */+type BaseErrorClass = new (description?: string) => BaseError;diff --git a/packages/shared/src/features/monitors/processor/processor.ts b/packages/shared/src/features/monitors/processor/processor.tsindex 0f857b03b1..ab9acb6677 100644--- a/packages/shared/src/features/monitors/processor/processor.ts+++ b/packages/shared/src/features/monitors/processor/processor.ts@@ -345,7 +345,7 @@ function processMonitor(args: {const automations = getAutomations({ monitor, completion, triggers });if (automations.length === 0) return [completion, []];- const alert = buildAlert({ prev: monitor, next: completion, runAt });+ const alert = buildAlert({ prev: monitor, next: completion, runAt, value });return [completion, toMonitorWebhookInputs({ alert, automations, now })];}@@ -380,8 +380,9 @@ function buildAlert(args: {prev: Monitor;next: MonitorCompletion;runAt: Date;+ value: number | null;}): MonitorAlert {- const { prev, next, runAt } = args;+ const { prev, next, runAt, value } = args;const { fromTimestamp, toTimestamp } = evaluationWindow(prev.window, runAt);return {monitorId: prev.id,@@ -407,6 +408,13 @@ function buildAlert(args: {view: prev.view,filters: prev.filters,window: prev.window,+ monitorName: prev.name,+ metric: prev.metric,+ metricValue: value,+ thresholdOperator: prev.thresholdOperator,+ alertThreshold: prev.alertThreshold,+ warningThreshold: prev.warningThreshold,+ alertedAt: next.alertedAt ?? undefined,};}diff --git a/packages/shared/src/features/monitors/types.ts b/packages/shared/src/features/monitors/types.tsindex 4ec25e1194..3a83365c87 100644--- a/packages/shared/src/features/monitors/types.ts+++ b/packages/shared/src/features/monitors/types.ts@@ -312,5 +312,12 @@ export const MonitorAlertSchema = z.object({view: MonitorViewSchema,filters: MonitorFiltersSchema,window: MonitorWindowSchema,+ monitorName: z.string().optional(),+ metric: MetricSchema.optional(),+ metricValue: z.number().nullable().optional(),+ thresholdOperator: MonitorThresholdOperatorSchema.optional(),+ alertThreshold: z.number().optional(),+ warningThreshold: z.number().nullable().optional(),+ alertedAt: z.coerce.date().optional(),});export type MonitorAlert = z.infer<typeof MonitorAlertSchema>;diff --git a/packages/shared/src/features/pagerduty/index.ts b/packages/shared/src/features/pagerduty/index.tsnew file mode 100644index 0000000000..9239830ba1--- /dev/null+++ b/packages/shared/src/features/pagerduty/index.ts@@ -0,0 +1,2 @@+/** index.ts is the client-safe barrel for the PagerDuty feature. */+export * from "./types";diff --git a/packages/shared/src/features/pagerduty/pagerDutyClient.ts b/packages/shared/src/features/pagerduty/pagerDutyClient.tsnew file mode 100644index 0000000000..0551c533c1--- /dev/null+++ b/packages/shared/src/features/pagerduty/pagerDutyClient.ts@@ -0,0 +1,153 @@+import { backOff } from "exponential-backoff";++import {+ GatewayTimeoutError,+ isBaseError,+ ServiceUnavailableError,+ TooManyRequestsError,+ toBaseError,+} from "../../errors";+import { logger } from "../../server/logger";+import { type PagerDutyRegion } from "./types";++/** PAGERDUTY_EVENTS_HOSTS is the fixed allowlist of Events API hosts per service region. */+export const PAGERDUTY_EVENTS_HOSTS: Record<PagerDutyRegion, string> = {+ US: "events.pagerduty.com",+ EU: "events.eu.pagerduty.com",+};++const summaryMaxLength = 1024;+const requestTimeoutMs = 10_000;+const maxAttempts = 4;++/** pagerDutyClient sends events to the Events API of a routing key's region, retrying 429, 5xx and network errors. */+export const pagerDutyClient = {+ /** alert enqueues an alert event at the region's `/v2/enqueue`. */+ alert(args: {+ region: PagerDutyRegion;+ event: PagerDutyAlertEvent;+ }): Promise<void> {+ return post(+ `https://${PAGERDUTY_EVENTS_HOSTS[args.region]}/v2/enqueue`,+ args.event,+ );+ },++ /** change posts a change event, which never pages anyone, to the region's `/v2/change/enqueue`. */+ change(args: {+ region: PagerDutyRegion;+ event: PagerDutyChangeEvent;+ }): Promise<void> {+ return post(+ `https://${PAGERDUTY_EVENTS_HOSTS[args.region]}/v2/change/enqueue`,+ args.event,+ );+ },+};++/** isRetryablePagerDutyError reports whether a failed Events API call may succeed on retry: 429, 5xx, timeouts and network errors. */+export function isRetryablePagerDutyError(error: unknown): boolean {+ if (error instanceof TooManyRequestsError) return true;+ return isBaseError(error) && error.httpCode >= 500;+}++/** truncatePagerDutySummary cuts a summary to the Events API limit. */+export function truncatePagerDutySummary(summary: string): string {+ if (summary.length <= summaryMaxLength) return summary;+ return `${summary.slice(0, summaryMaxLength - 3)}...`;+}++/** post sends one event to an Events API URL, retrying retryable failures. */+async function post(+ url: string,+ body: PagerDutyAlertEvent | PagerDutyChangeEvent,+): Promise<void> {+ await backOff(() => postOnce(url, body), {+ numOfAttempts: maxAttempts,+ retry: shouldRetry,+ });+}++/** shouldRetry retries retryable failures and logs each retry. */+function shouldRetry(error: unknown): boolean {+ if (!isRetryablePagerDutyError(error)) return false;+ logger.warn(`Retrying PagerDuty event: ${String(error)}`);+ return true;+}++/** postOnce sends one event once and throws the base error matching a rejection. */+async function postOnce(+ url: string,+ body: PagerDutyAlertEvent | PagerDutyChangeEvent,+): Promise<void> {+ const response = await fetchEvent(url, body);+ if (response.ok) return;++ const responseBody = (await response.text().catch(() => "")).slice(0, 1000);+ throw toBaseError(+ response.status,+ `PagerDuty returned ${response.status}: ${responseBody}`.trim(),+ );+}++/** fetchEvent posts the JSON body, turning timeouts into GatewayTimeoutErrors and other network errors into ServiceUnavailableErrors. */+async function fetchEvent(+ url: string,+ body: PagerDutyAlertEvent | PagerDutyChangeEvent,+): Promise<Response> {+ try {+ return await fetch(url, {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ body: JSON.stringify(body),+ redirect: "error",+ signal: AbortSignal.timeout(requestTimeoutMs),+ });+ } catch (error) {+ if (error instanceof Error && error.name === "TimeoutError") {+ throw new GatewayTimeoutError(+ `PagerDuty did not respond within ${requestTimeoutMs}ms`,+ );+ }+ const reason = error instanceof Error ? error.message : "Unknown error";+ throw new ServiceUnavailableError(`PagerDuty request failed: ${reason}`);+ }+}++/** PagerDutyLink is a clickable link rendered on the PagerDuty alert. */+export type PagerDutyLink = { href: string; text?: string };++/** PagerDutyAlertEvent is an Events API v2 alert event. */+export type PagerDutyAlertEvent = {+ routing_key: string;+ event_action: "trigger" | "acknowledge" | "resolve";+ dedup_key: string;+ client?: string;+ client_url?: string;+ links?: PagerDutyLink[];+ payload?: {+ summary: string;+ source: string;+ severity: PagerDutySeverity;+ timestamp?: string;+ component?: string;+ group?: string;+ class?: string;+ custom_details?: Record<string, unknown>;+ };+};++/** PagerDutySeverity is the alert severity PagerDuty routes on. */+export type PagerDutySeverity = "critical" | "error" | "warning" | "info";++/** PagerDutyChangeEvent is a Change Events API event. */+export type PagerDutyChangeEvent = {+ routing_key: string;+ links?: PagerDutyLink[];+ payload: {+ summary: string;+ source?: string;+ timestamp?: string;+ custom_details?: Record<string, unknown>;+ };+};diff --git a/packages/shared/src/features/pagerduty/server.ts b/packages/shared/src/features/pagerduty/server.tsnew file mode 100644index 0000000000..0b8b29922a--- /dev/null+++ b/packages/shared/src/features/pagerduty/server.ts@@ -0,0 +1,4 @@+/** server.ts is the server-only barrel for the PagerDuty feature. It+ * re-exports everything client-safe from `./index` plus the Events API client. */+export * from "./index";+export * from "./pagerDutyClient";diff --git a/packages/shared/src/features/pagerduty/types.ts b/packages/shared/src/features/pagerduty/types.tsnew file mode 100644index 0000000000..9c30d7950e--- /dev/null+++ b/packages/shared/src/features/pagerduty/types.ts@@ -0,0 +1,31 @@+import { z } from "zod";++/** PagerDutyRegionSchema is the service region of a PagerDuty account; it picks the Events API host. */+export const PagerDutyRegionSchema = z.enum(["US", "EU"]);++/**+ * PagerDutyRoutingKeySchema accepts an Events API v2 routing key: a service+ * integration key or an Event Orchestration key. PagerDuty issues both as 32+ * alphanumeric characters.+ */+export const PagerDutyRoutingKeySchema = z+ .string()+ .trim()+ .regex(+ /^[A-Za-z0-9]{32}$/,+ "Routing key must be the 32-character integration key from PagerDuty",+ );++/** PagerDutyTestEventStepSchema is a step of the test alert: trigger it at critical severity, or resolve it. */+export const PagerDutyTestEventStepSchema = z.enum(["trigger", "resolve"]);++/** maskPagerDutyRoutingKey keeps the first and last four characters, e.g. `R0ab...9xYz`. */+export function maskPagerDutyRoutingKey(routingKey: string): string {+ if (routingKey.length <= 8) return "...";+ return `${routingKey.slice(0, 4)}...${routingKey.slice(-4)}`;+}++export type PagerDutyRegion = z.infer<typeof PagerDutyRegionSchema>;+export type PagerDutyTestEventStep = z.infer<+ typeof PagerDutyTestEventStepSchema+>;diff --git a/packages/shared/src/server/repositories/automation-repository.ts b/packages/shared/src/server/repositories/automation-repository.tsindex 1efe76c0a0..36219d928f 100644--- a/packages/shared/src/server/repositories/automation-repository.ts+++ b/packages/shared/src/server/repositories/automation-repository.ts@@ -26,6 +26,8 @@ import {SafeWebhookActionConfigSchema,SlackActionConfigSchema,SafeGitHubDispatchActionConfigSchema,+ isPagerDutyActionConfig,+ SafePagerDutyActionConfigSchema,} from "../../domain/automations";import { InternalServerError } from "../../errors";import { FilterState } from "../../types";@@ -78,7 +80,7 @@ export const getActionByIdWithSecrets = async ({};}- // SLACK and GITHUB_DISPATCH need no header decryption, so the stored config+ // SLACK, GITHUB_DISPATCH and PAGERDUTY need no header decryption, so the stored config// is returned verbatim. Secrets are included by design here - callers that// hand configs to a client must use getActionById instead.return actionConfig as ActionDomainWithSecrets;@@ -240,6 +242,19 @@ const convertToSafeActionConfig = (action: Action): SafeActionConfig => {actionType,Object.keys(SafeGitHubDispatchActionConfigSchema.shape),);+ case "PAGERDUTY":+ if (isPagerDutyActionConfig(action.config)) {+ return {+ type: action.config.type,+ displayRoutingKey: action.config.displayRoutingKey,+ region: action.config.region,+ };+ }+ return pickSafeConfigFields(+ action.config,+ actionType,+ Object.keys(SafePagerDutyActionConfigSchema.shape),+ );default: {const unhandledActionType: never = actionType;throw new InternalServerError(diff --git a/web/src/__tests__/server/automations-trpc.servertest.ts b/web/src/__tests__/server/automations-trpc.servertest.tsindex 98e1a1c78a..aa0c381159 100644--- a/web/src/__tests__/server/automations-trpc.servertest.ts+++ b/web/src/__tests__/server/automations-trpc.servertest.ts@@ -23,6 +23,8 @@ import {generateWebhookSecret,} from "@langfuse/shared/encryption";import { TRPCError } from "@trpc/server";+import { type MockInstance } from "vitest";+import { env } from "@/src/env.mjs";// Fake Slack workspace: the bot is in private channels "G_PRIVATE" and archived// "G_ARCHIVED". Slack API calls are faked; the access logic is real.@@ -3387,4 +3389,327 @@ describe("automations trpc", () => {expect(config.eventType).toBe("langfuse-prompt-created");});});+ describe("PagerDuty actions", () => {+ const routingKey = "R0abcdefghijklmnopqrstuvwxyz9xYz";+ const newRoutingKey = "N3wkeyabcdefghijklmnopqrstuv7uVw";+ const mutableEnv = env as { NEXT_PUBLIC_LANGFUSE_CLOUD_REGION?: string };+ const originalCloudRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;++ beforeAll(() => {+ mutableEnv.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "STAGING";+ });++ afterAll(() => {+ mutableEnv.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = originalCloudRegion;+ });++ const createInput = (+ projectId: string,+ actionConfig: {+ routingKey?: string;+ region: "US" | "EU";+ } = { routingKey, region: "US" },+ ) => ({+ projectId,+ name: "Page on-call",+ eventSource: TriggerEventSource.Monitor,+ eventAction: [],+ filter: [],+ status: JobConfigState.ACTIVE,+ actionType: "PAGERDUTY" as const,+ actionConfig: { type: "PAGERDUTY" as const, ...actionConfig },+ });++ async function storedConfig(actionId: string) {+ const action = await prisma.action.findUniqueOrThrow({+ where: { id: actionId },+ });+ return action.config as Record<string, string>;+ }++ function viewerCaller(session: Session) {+ const viewerSession: Session = {+ ...session,+ user: {+ ...session.user!,+ admin: false,+ organizations: [+ {+ ...session.user!.organizations[0],+ role: "MEMBER",+ projects: [+ {+ ...session.user!.organizations[0].projects[0],+ role: "VIEWER",+ },+ ],+ },+ ],+ },+ };+ return appRouter.createCaller({+ ...createInnerTRPCContext({ session: viewerSession, headers: {} }),+ prisma,+ });+ }++ it("encrypts the routing key and returns only the masked key", async () => {+ const { project, caller } = await prepare();++ const result = await caller.automations.createAutomation(+ createInput(project.id, { routingKey, region: "EU" }),+ );++ const config = await storedConfig(result.action.id);+ expect(config.routingKey).not.toBe(routingKey);+ expect(decrypt(config.routingKey)).toBe(routingKey);+ expect(config.displayRoutingKey).toBe("R0ab...9xYz");+ expect(config.region).toBe("EU");+ expect(result.action.config).toEqual({+ type: "PAGERDUTY",+ displayRoutingKey: "R0ab...9xYz",+ region: "EU",+ });+ });++ function callerWithUser(session: Session, user: Partial<Session["user"]>) {+ return appRouter.createCaller({+ ...createInnerTRPCContext({+ session: { ...session, user: { ...session.user!, ...user } },+ headers: {},+ }),+ prisma,+ });+ }++ it("rejects users who are not Langfuse Cloud admins", async () => {+ const { project, session } = await prepare();+ const owner = callerWithUser(session, { admin: false });++ await expect(+ owner.automations.createAutomation(createInput(project.id)),+ ).rejects.toMatchObject({ code: "FORBIDDEN" });+ await expect(+ owner.automations.sendPagerDutyTestEvent({+ projectId: project.id,+ region: "US",+ routingKey,+ step: "trigger",+ }),+ ).rejects.toMatchObject({ code: "FORBIDDEN" });+ });++ it("rejects admins on self-hosted instances", async () => {+ const { project, caller } = await prepare();+ mutableEnv.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;++ try {+ await expect(+ caller.automations.createAutomation(createInput(project.id)),+ ).rejects.toMatchObject({ code: "FORBIDDEN" });+ } finally {+ mutableEnv.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "STAGING";+ }+ });++ it("rejects a create without a routing key", async () => {+ const { project, caller } = await prepare();++ await expect(+ caller.automations.createAutomation(+ createInput(project.id, { region: "US" }),+ ),+ ).rejects.toMatchObject({ code: "BAD_REQUEST" });+ });++ it("rejects a malformed routing key", async () => {+ const { project, caller } = await prepare();++ await expect(+ caller.automations.createAutomation(+ createInput(project.id, { routingKey: "too-short", region: "US" }),+ ),+ ).rejects.toMatchObject({ code: "BAD_REQUEST" });+ });++ it("keeps the stored key when an update omits it", async () => {+ const { project, caller } = await prepare();+ const created = await caller.automations.createAutomation(+ createInput(project.id),+ );++ await caller.automations.updateAutomation({+ ...createInput(project.id, { region: "EU" }),+ automationId: created.automation.id,+ });++ const config = await storedConfig(created.action.id);+ expect(decrypt(config.routingKey)).toBe(routingKey);+ expect(config.displayRoutingKey).toBe("R0ab...9xYz");+ expect(config.region).toBe("EU");+ });++ it("replaces the key when an update sends a new one", async () => {+ const { project, caller } = await prepare();+ const created = await caller.automations.createAutomation(+ createInput(project.id),+ );++ await caller.automations.updateAutomation({+ ...createInput(project.id, {+ routingKey: newRoutingKey,+ region: "US",+ }),+ automationId: created.automation.id,+ });++ const config = await storedConfig(created.action.id);+ expect(decrypt(config.routingKey)).toBe(newRoutingKey);+ expect(config.displayRoutingKey).toBe("N3wk...7uVw");+ });++ it("never returns the routing key on any read path to a project VIEWER", async () => {+ const { project, session, caller } = await prepare();+ const created = await caller.automations.createAutomation(+ createInput(project.id),+ );+ const storedKey = (await storedConfig(created.action.id)).routingKey;+ const viewer = viewerCaller(session);++ const responses = [+ await viewer.automations.getAutomations({ projectId: project.id }),+ await viewer.automations.getAutomation({+ projectId: project.id,+ automationId: created.automation.id,+ }),+ ];++ for (const response of responses) {+ const serialized = JSON.stringify(response);+ expect(serialized).not.toContain(routingKey);+ expect(serialized).not.toContain(storedKey);+ expect(serialized).not.toContain("routingKey");+ }+ });++ describe("sendPagerDutyTestEvent", () => {+ let fetchSpy: MockInstance<typeof fetch>;++ beforeEach(() => {+ fetchSpy = vi+ .spyOn(global, "fetch")+ .mockResolvedValue(new Response("{}", { status: 202 }));+ });++ afterEach(() => {+ fetchSpy.mockRestore();+ });++ function sentEvents() {+ return fetchSpy.mock.calls.map(([url, init]) => ({+ url: String(url),+ body: JSON.parse(String((init as RequestInit).body)),+ }));+ }++ it.each([+ ["trigger", "trigger", "critical"],+ ["resolve", "resolve", undefined],+ ] as const)(+ "sends the %s step as one %s under the project's test key",+ async (step, eventAction, severity) => {+ const { project, caller } = await prepare();++ await caller.automations.sendPagerDutyTestEvent({+ projectId: project.id,+ region: "EU",+ routingKey,+ step,+ });++ const [event, ...rest] = sentEvents();+ expect(rest).toEqual([]);+ expect(event.url).toBe("https://events.eu.pagerduty.com/v2/enqueue");+ expect(event.body).toMatchObject({+ routing_key: routingKey,+ event_action: eventAction,+ dedup_key: `langfuse-${project.id}-test`,+ });+ expect(event.body.payload?.severity).toBe(severity);+ },+ );++ it("falls back to the stored key of the action", async () => {+ const { project, caller } = await prepare();+ const created = await caller.automations.createAutomation(+ createInput(project.id),+ );++ await caller.automations.sendPagerDutyTestEvent({+ projectId: project.id,+ region: "US",+ actionId: created.action.id,+ step: "trigger",+ });++ expect(sentEvents()[0]).toMatchObject({+ url: "https://events.pagerduty.com/v2/enqueue",+ body: { routing_key: routingKey },+ });+ });++ it("rejects a stored key of another project", async () => {+ const { project: otherProject, caller: otherCaller } = await prepare();+ const otherAction = await otherCaller.automations.createAutomation(+ createInput(otherProject.id),+ );+ const { project, caller } = await prepare();++ await expect(+ caller.automations.sendPagerDutyTestEvent({+ projectId: project.id,+ region: "US",+ actionId: otherAction.action.id,+ step: "trigger",+ }),+ ).rejects.toMatchObject({ code: "BAD_REQUEST" });+ expect(fetchSpy).not.toHaveBeenCalled();+ });++ it("maps a PagerDuty rejection to BAD_REQUEST", async () => {+ fetchSpy.mockResolvedValue(+ new Response("invalid key", { status: 400 }),+ );+ const { project, caller } = await prepare();++ await expect(+ caller.automations.sendPagerDutyTestEvent({+ projectId: project.id,+ region: "US",+ routingKey,+ step: "trigger",+ }),+ ).rejects.toMatchObject({ code: "BAD_REQUEST" });+ });++ it.each([+ [429, "TOO_MANY_REQUESTS"],+ [502, "BAD_GATEWAY"],+ [503, "SERVICE_UNAVAILABLE"],+ ])("returns a PagerDuty %i as %s after retries", async (status, code) => {+ fetchSpy.mockResolvedValue(new Response("busy", { status }));+ const { project, caller } = await prepare();++ await expect(+ caller.automations.sendPagerDutyTestEvent({+ projectId: project.id,+ region: "US",+ routingKey,+ step: "trigger",+ }),+ ).rejects.toMatchObject({ code });+ expect(fetchSpy).toHaveBeenCalledTimes(4);+ });+ });+ });});diff --git a/web/src/__tests__/server/slack-integration.servertest.ts b/web/src/__tests__/server/slack-integration.servertest.tsindex ff5342e400..8f2ad0b8c4 100644--- a/web/src/__tests__/server/slack-integration.servertest.ts+++ b/web/src/__tests__/server/slack-integration.servertest.ts@@ -690,6 +690,25 @@ describe("Slack Integration", () => {caller.slack.disconnect({ projectId: project.id }),).rejects.toThrow("Slack integration not found");});++ it("should reject MEMBER, who may not manage integrations", async () => {+ const { caller, project } = await prepare({+ projectRole: "MEMBER",+ admin: false,+ });+ await createIntegration(project.id, null);+ mockSlackService.deleteIntegration.mockClear();++ await expect(+ caller.slack.disconnect({ projectId: project.id }),+ ).rejects.toMatchObject({ code: "FORBIDDEN" });+ expect(mockSlackService.deleteIntegration).not.toHaveBeenCalled();+ expect(+ await prisma.slackIntegration.findUnique({+ where: { projectId: project.id },+ }),+ ).not.toBeNull();+ });});});diff --git a/web/src/features/automations/actionTypeLabels.ts b/web/src/features/automations/actionTypeLabels.tsnew file mode 100644index 0000000000..4968c9bf8a--- /dev/null+++ b/web/src/features/automations/actionTypeLabels.ts@@ -0,0 +1,9 @@+import { type ActionTypes } from "@langfuse/shared";++/** actionTypeLabel maps each automation action type to its display name. */+export const actionTypeLabel: Record<ActionTypes, string> = {+ WEBHOOK: "Webhook",+ SLACK: "Slack",+ GITHUB_DISPATCH: "GitHub Dispatch",+ PAGERDUTY: "PagerDuty",+};diff --git a/web/src/features/automations/components/AutomationForm.tsx b/web/src/features/automations/components/AutomationForm.tsxindex 41eb98a2c9..e92577ca60 100644--- a/web/src/features/automations/components/AutomationForm.tsx+++ b/web/src/features/automations/components/AutomationForm.tsx@@ -44,11 +44,17 @@ import {TriggerEventSourceSchema,webhookActionFilterOptions,} from "@langfuse/shared";+import {+ PagerDutyRegionSchema,+ PagerDutyRoutingKeySchema,+} from "@langfuse/shared/pagerduty";import { InlineFilterBuilder, MultiSelect } from "@/src/features/filters";import { DeleteAutomationDialogController } from "./DeleteAutomationDialogController";import { useLangfuseCloudRegion } from "@/src/features/organizations";import { useHasProjectAccess } from "@/src/features/rbac";import { ActionHandlerRegistry } from "./actions";+import { actionTypeLabel } from "@/src/features/automations/actionTypeLabels";+import { useAvailableActionTypes } from "@/src/features/automations/useAvailableActionTypes";import { webhookSchema } from "./actions/WebhookActionForm";import { Alert } from "@/src/components/design-system/Alert/Alert";import Link from "next/link";@@ -70,6 +76,13 @@ const githubDispatchSchema = z.object({originalUrl: z.string().optional(),});+// Define PagerDuty action schema; an empty routing key keeps the stored one on edit+const pagerDutySchema = z.object({+ routingKey: z.union([z.literal(""), PagerDutyRoutingKeySchema]),+ displayRoutingKey: z.string().optional(),+ region: PagerDutyRegionSchema,+});+/** promptEventActionDefaults is the default eventAction set for a fresh prompt-source automation. */const promptEventActionDefaults: string[] = ["created", "updated", "deleted"];@@ -78,6 +91,7 @@ const projectNotificationName = (data: FormValues): string => {// Automation names are visible to every project member, so they must not// carry Slack channel names (private ones are only for channel members).if (data.actionType === "SLACK") return "Slack";+ if (data.actionType === "PAGERDUTY") return "PagerDuty";if (data.actionType === "WEBHOOK") {try {return `Webhook ${new URL(data.webhook.url).hostname}`;@@ -195,6 +209,10 @@ const formSchema = zactionType: z.literal("GITHUB_DISPATCH"),githubDispatch: githubDispatchSchema,}),+ baseFormSchema.extend({+ actionType: z.literal("PAGERDUTY"),+ pagerDuty: pagerDutySchema,+ }),]).superRefine((data, ctx) => {// Prompt-source triggers require at least one event action; monitor and@@ -408,6 +426,9 @@ export const AutomationForm = ({projectId,scope: "automations:CUD",});+ const availableActionTypes = useAvailableActionTypes(+ allowedActionTypes ?? ActionHandlerRegistry.getAllActionTypes(),+ );const utils = api.useUtils();@@ -522,6 +543,14 @@ export const AutomationForm = ({originalUrl: githubDefaults.githubDispatch.originalUrl,},};+ } else if (actionType === "PAGERDUTY") {+ const handler = ActionHandlerRegistry.getHandler("PAGERDUTY");+ const pagerDutyDefaults = handler.getDefaultValues(automation);+ return {+ ...baseValues,+ actionType: "PAGERDUTY" as const,+ pagerDuty: pagerDutyDefaults.pagerDuty,+ };}throw new Error("Invalid action type");};@@ -645,6 +674,10 @@ export const AutomationForm = ({const handler = ActionHandlerRegistry.getHandler("GITHUB_DISPATCH");const defaultValues = handler.getDefaultValues();form.setValue("githubDispatch", defaultValues.githubDispatch);+ } else if (value === "PAGERDUTY") {+ const handler = ActionHandlerRegistry.getHandler("PAGERDUTY");+ const defaultValues = handler.getDefaultValues();+ form.setValue("pagerDuty", defaultValues.pagerDuty);}};@@ -798,18 +831,9 @@ export const AutomationForm = ({</SelectTrigger></FormControl><SelectContent>- {(- allowedActionTypes ??- ActionHandlerRegistry.getAllActionTypes()- ).map((actionType) => (+ {availableActionTypes.map((actionType) => (<SelectItem key={actionType} value={actionType}>- {actionType === "WEBHOOK"- ? "Webhook"- : actionType === "SLACK"- ? "Slack"- : actionType === "GITHUB_DISPATCH"- ? "GitHub Dispatch"- : "Annotation Queue"}+ {actionTypeLabel[actionType]}</SelectItem>))}<SelectItem disabled={true} value="planned">diff --git a/web/src/features/automations/components/AutomationSidebar.tsx b/web/src/features/automations/components/AutomationSidebar.tsxindex 1a2a26b8c6..d4362f4a54 100644--- a/web/src/features/automations/components/AutomationSidebar.tsx+++ b/web/src/features/automations/components/AutomationSidebar.tsx@@ -1,4 +1,3 @@-/* eslint-disable no-nested-ternary */import React from "react";import { api } from "@/src/utils/api";import {@@ -8,6 +7,7 @@ import {} from "@langfuse/shared";import { cn } from "@/src/utils/tailwind";import { StatusBadge } from "@/src/components/ui/StatusBadge/StatusBadge";+import { actionTypeLabel } from "@/src/features/automations/actionTypeLabels";interface AutomationSidebarProps {projectId: string;@@ -113,11 +113,7 @@ export const AutomationSidebar: React.FC<AutomationSidebarProps> = ({: automation.trigger.eventSource}</span>{" → "}- {automation.action.type === "WEBHOOK"- ? "Webhook"- : automation.action.type === "SLACK"- ? "Slack"- : "Annotation Queue"}+ {actionTypeLabel[automation.action.type]}</p></div></div>diff --git a/web/src/features/automations/components/actions/ActionHandlerRegistry.ts b/web/src/features/automations/components/actions/ActionHandlerRegistry.tsindex e9f99aa491..02806833af 100644--- a/web/src/features/automations/components/actions/ActionHandlerRegistry.ts+++ b/web/src/features/automations/components/actions/ActionHandlerRegistry.ts@@ -4,6 +4,7 @@ import { type BaseActionHandler } from "./BaseActionHandler";import { WebhookActionHandler } from "./WebhookActionHandler";import { SlackActionHandler } from "./SlackActionHandler";import { GitHubDispatchActionHandler } from "./GitHubDispatchActionHandler";+import { PagerDutyActionHandler } from "./PagerDutyActionHandler";export class ActionHandlerRegistry {private static handlers: Map<ActionType, BaseActionHandler<FieldValues>> =@@ -14,6 +15,7 @@ export class ActionHandlerRegistry {this.handlers.set("WEBHOOK", new WebhookActionHandler());this.handlers.set("SLACK", new SlackActionHandler());this.handlers.set("GITHUB_DISPATCH", new GitHubDispatchActionHandler());+ this.handlers.set("PAGERDUTY", new PagerDutyActionHandler());}static getHandler<T extends FieldValues = FieldValues>(diff --git a/web/src/features/automations/components/actions/PagerDutyActionHandler.ts b/web/src/features/automations/components/actions/PagerDutyActionHandler.tsnew file mode 100644index 0000000000..06cc87c047--- /dev/null+++ b/web/src/features/automations/components/actions/PagerDutyActionHandler.ts@@ -0,0 +1,73 @@+import React from "react";+import { type UseFormReturn } from "react-hook-form";++import {+ type ActionCreate,+ type AutomationDomain,+ type SafePagerDutyActionConfig,+} from "@langfuse/shared";+import { type PagerDutyRegion } from "@langfuse/shared/pagerduty";+import { PagerDutyActionForm } from "@/src/features/pagerduty";+import { type BaseActionHandler } from "./BaseActionHandler";++/** PagerDutyActionHandler wires the PagerDuty action into the automation form. */+export class PagerDutyActionHandler implements BaseActionHandler<PagerDutyActionFormData> {+ actionType = "PAGERDUTY" as const;++ getDefaultValues(automation?: AutomationDomain): PagerDutyActionFormData {+ const config = storedConfig(automation);+ return {+ pagerDuty: {+ routingKey: "", // never populated with the stored key+ displayRoutingKey: config?.displayRoutingKey,+ region: config?.region ?? "US",+ },+ };+ }++ validateFormData(formData: PagerDutyActionFormData): {+ isValid: boolean;+ errors?: string[];+ } {+ const { routingKey, displayRoutingKey } = formData.pagerDuty ?? {};+ if (routingKey?.trim() || displayRoutingKey) return { isValid: true };+ return { isValid: false, errors: ["PagerDuty routing key is required"] };+ }++ buildActionConfig(formData: PagerDutyActionFormData): ActionCreate {+ const routingKey = formData.pagerDuty.routingKey.trim() || undefined;+ return { type: "PAGERDUTY", region: formData.pagerDuty.region, routingKey };+ }++ renderForm(props: {+ form: UseFormReturn<PagerDutyActionFormData>;+ disabled: boolean;+ projectId: string;+ action?: AutomationDomain["action"];+ }) {+ return React.createElement(PagerDutyActionForm, {+ form: props.form,+ disabled: props.disabled,+ projectId: props.projectId,+ actionId: props.action?.id,+ });+ }+}++/** storedConfig returns the saved PagerDuty config of an automation, if any. */+function storedConfig(+ automation?: AutomationDomain,+): SafePagerDutyActionConfig | undefined {+ const config = automation?.action?.config;+ if (config?.type !== "PAGERDUTY") return undefined;+ return config;+}++/** PagerDutyActionFormData is the PagerDuty part of the automation form. */+type PagerDutyActionFormData = {+ pagerDuty: {+ routingKey: string;+ displayRoutingKey?: string;+ region: PagerDutyRegion;+ };+};diff --git a/web/src/features/automations/index.ts b/web/src/features/automations/index.tsindex 5c8f13094d..eeb68526c7 100644--- a/web/src/features/automations/index.ts+++ b/web/src/features/automations/index.ts@@ -8,4 +8,6 @@ export {AutomationForm,automationCreateHref,} from "@/src/features/automations/components/AutomationForm";+export { actionTypeLabel } from "@/src/features/automations/actionTypeLabels";+export { useAvailableActionTypes } from "@/src/features/automations/useAvailableActionTypes";export { WebhookSecretRender } from "@/src/features/automations/components/WebhookSecretRender";diff --git a/web/src/features/automations/server/router.ts b/web/src/features/automations/server/router.tsindex 71e3f756ca..695c2346df 100644--- a/web/src/features/automations/server/router.ts+++ b/web/src/features/automations/server/router.ts@@ -13,6 +13,11 @@ import {TriggerEventSourceSchema,ProjectNotificationEventTypeSchema,} from "@langfuse/shared";+import {+ PagerDutyRegionSchema,+ PagerDutyRoutingKeySchema,+ PagerDutyTestEventStepSchema,+} from "@langfuse/shared/pagerduty";import { throwIfNoProjectAccess } from "@/src/features/rbac";import { v4 } from "uuid";import {@@ -27,6 +32,11 @@ import {import { generateWebhookSecret, encrypt } from "@langfuse/shared/encryption";import { processWebhookActionConfig } from "./webhookHelpers";import { processGitHubDispatchActionConfig } from "./githubDispatchHelpers";+import {+ processPagerDutyActionConfig,+ sendPagerDutyTestEvent,+ throwIfPagerDutyUnavailable,+} from "@/src/features/pagerduty/server";import { maskActionForViewer, processSlackActionConfig } from "./slackHelpers";import { updateTriggerEventActions } from "./automationService";import { TRPCError } from "@trpc/server";@@ -329,6 +339,12 @@ export const automationsRouter = createTRPCRouter({});finalActionConfig = githubResult.finalActionConfig;newUnencryptedWebhookSecret = githubResult.githubToken;+ } else if (input.actionType === "PAGERDUTY") {+ throwIfPagerDutyUnavailable(ctx.session);+ finalActionConfig = await processPagerDutyActionConfig({+ actionConfig: input.actionConfig,+ projectId: input.projectId,+ });}const [trigger, action, automation] = await ctx.prisma.$transaction(@@ -440,6 +456,13 @@ export const automationsRouter = createTRPCRouter({projectId: input.projectId,});finalActionConfig = githubResult.finalActionConfig;+ } else if (input.actionType === "PAGERDUTY") {+ throwIfPagerDutyUnavailable(ctx.session);+ finalActionConfig = await processPagerDutyActionConfig({+ actionConfig: input.actionConfig,+ actionId: existingAutomation.action.id,+ projectId: input.projectId,+ });}const [action, trigger, automation] = await ctx.prisma.$transaction(@@ -650,4 +673,36 @@ export const automationsRouter = createTRPCRouter({return count;}),++ sendPagerDutyTestEvent: protectedProjectProcedure+ .input(+ z.object({+ projectId: z.string(),+ region: PagerDutyRegionSchema,+ routingKey: PagerDutyRoutingKeySchema.optional(),+ actionId: z.string().optional(),+ step: PagerDutyTestEventStepSchema,+ }),+ )+ .mutation(async ({ ctx, input }) => {+ throwIfNoProjectAccess({+ session: ctx.session,+ projectId: input.projectId,+ scope: "automations:CUD",+ });++ throwIfPagerDutyUnavailable(ctx.session);+ await sendPagerDutyTestEvent(input);++ if (input.actionId) {+ await auditLog({+ session: ctx.session,+ resourceType: "action",+ resourceId: input.actionId,+ action: "create",+ after: { action: "test_event_sent", region: input.region },+ });+ }+ return { success: true };+ }),});diff --git a/web/src/features/automations/useAvailableActionTypes.ts b/web/src/features/automations/useAvailableActionTypes.tsnew file mode 100644index 0000000000..f58ab7e22b--- /dev/null+++ b/web/src/features/automations/useAvailableActionTypes.ts@@ -0,0 +1,20 @@+import { useSession } from "next-auth/react";++import { type ActionTypes } from "@langfuse/shared";+import { useLangfuseCloudRegion } from "@/src/features/organizations";++const cloudAdminActionTypes: readonly ActionTypes[] = ["PAGERDUTY"];++/** useAvailableActionTypes drops action types limited to Langfuse Cloud admins for everyone else. */+export function useAvailableActionTypes(+ actionTypes: readonly ActionTypes[],+): ActionTypes[] {+ const session = useSession();+ const { isLangfuseCloud } = useLangfuseCloudRegion();+ if (isLangfuseCloud && session.data?.user?.admin === true) {+ return [...actionTypes];+ }+ return actionTypes.filter(+ (actionType) => !cloudAdminActionTypes.includes(actionType),+ );+}diff --git a/web/src/features/monitors/components/MonitorAutomationsPanel.access.clienttest.tsx b/web/src/features/monitors/components/MonitorAutomationsPanel.access.clienttest.tsxindex 101f5cd50f..22078a70fc 100644--- a/web/src/features/monitors/components/MonitorAutomationsPanel.access.clienttest.tsx+++ b/web/src/features/monitors/components/MonitorAutomationsPanel.access.clienttest.tsx@@ -7,6 +7,10 @@ const automationRow = {action: { type: "WEBHOOK" },};+vi.mock("next-auth/react", () => ({+ useSession: () => ({ data: null, status: "unauthenticated" }),+}));+vi.mock("@/src/utils/api", () => ({api: {automations: {diff --git a/web/src/features/monitors/components/MonitorAutomationsPanel.create.clienttest.tsx b/web/src/features/monitors/components/MonitorAutomationsPanel.create.clienttest.tsxindex b1e5c431bb..cb9a1f3248 100644--- a/web/src/features/monitors/components/MonitorAutomationsPanel.create.clienttest.tsx+++ b/web/src/features/monitors/components/MonitorAutomationsPanel.create.clienttest.tsx@@ -29,6 +29,10 @@ let automations: (typeof existingRow)[] = [];/** gateFetch defers the post-create list read so a test can interleave user input. */let gateFetch: Promise<unknown> | undefined;+vi.mock("next-auth/react", () => ({+ useSession: () => ({ data: null, status: "unauthenticated" }),+}));+vi.mock("@/src/utils/api", () => ({api: {automations: {diff --git a/web/src/features/monitors/components/MonitorAutomationsPanel.prune.clienttest.tsx b/web/src/features/monitors/components/MonitorAutomationsPanel.prune.clienttest.tsxindex 2383f68018..ecaeac1a3a 100644--- a/web/src/features/monitors/components/MonitorAutomationsPanel.prune.clienttest.tsx+++ b/web/src/features/monitors/components/MonitorAutomationsPanel.prune.clienttest.tsx@@ -9,6 +9,10 @@ const automationRow = {const useQueryMock = vi.fn();+vi.mock("next-auth/react", () => ({+ useSession: () => ({ data: null, status: "unauthenticated" }),+}));+vi.mock("@/src/utils/api", () => ({api: {automations: {diff --git a/web/src/features/monitors/components/MonitorAutomationsPanel.toggle.clienttest.tsx b/web/src/features/monitors/components/MonitorAutomationsPanel.toggle.clienttest.tsxindex 8d02164784..b6a3f2df32 100644--- a/web/src/features/monitors/components/MonitorAutomationsPanel.toggle.clienttest.tsx+++ b/web/src/features/monitors/components/MonitorAutomationsPanel.toggle.clienttest.tsx@@ -9,6 +9,10 @@ const automationRow = {const useQueryMock = vi.fn();+vi.mock("next-auth/react", () => ({+ useSession: () => ({ data: null, status: "unauthenticated" }),+}));+vi.mock("@/src/utils/api", () => ({api: {automations: {diff --git a/web/src/features/monitors/components/MonitorAutomationsPanel.tsx b/web/src/features/monitors/components/MonitorAutomationsPanel.tsxindex 51cec8f148..60ec599f2b 100644--- a/web/src/features/monitors/components/MonitorAutomationsPanel.tsx+++ b/web/src/features/monitors/components/MonitorAutomationsPanel.tsx@@ -7,6 +7,7 @@ import {Slack,Webhook as WebhookIcon,} from "@langfuse/icons";+import { SiPagerduty } from "@langfuse/icons/si";import { api } from "@/src/utils/api";import { Button } from "@/src/components/ui/button";@@ -22,7 +23,9 @@ import {} from "@/src/components/ui/dialog";import { DropdownMenu } from "@/src/components/design-system/DropdownMenu/DropdownMenu";import {+ actionTypeLabel,AutomationForm,+ useAvailableActionTypes,WebhookSecretRender,} from "@/src/features/automations";import { cn } from "@/src/utils/tailwind";@@ -33,17 +36,11 @@ import {TriggerEventSource,} from "@langfuse/shared";-/** actionLabel maps each automation action type to its display name. */-const actionLabel: Record<ActionTypes, string> = {- WEBHOOK: "Webhook",- SLACK: "Slack",- GITHUB_DISPATCH: "GitHub Dispatch",-};-const actionIcon = {WEBHOOK: WebhookIcon,SLACK: Slack,GITHUB_DISPATCH: Github,+ PAGERDUTY: SiPagerduty,} satisfies Record<ActionTypes, typeof WebhookIcon>;/** MonitorAutomationsPanel lets the user select which automations fire for a monitor via explicit trigger IDs. */@@ -271,6 +268,7 @@ const AddAutomationDropdown = ({isDisabled?: boolean;onAutomationCreated: (automationId: string) => void;}) => {+ const actionTypes = useAvailableActionTypes(ActionTypeSchema.options);const [draft, setDraft] = useState<NewAutomationDraft | null>(null);const [createdSecret, setCreatedSecret] =useState<CreatedWebhookSecret | null>(null);@@ -295,10 +293,10 @@ const AddAutomationDropdown = ({icon: Plus,onClick: () => setDraft({}),},- ...ActionTypeSchema.options.map((actionType) => ({+ ...actionTypes.map((actionType) => ({type: "item" as const,id: actionType,- title: actionLabel[actionType],+ title: actionTypeLabel[actionType],icon: actionIcon[actionType],onClick: () => setDraft({ actionType }),})),@@ -403,5 +401,7 @@ const ActionIcon = ({return <Slack __rawClassName={className} />;case "GITHUB_DISPATCH":return <Github __rawClassName={className} />;+ case "PAGERDUTY":+ return <SiPagerduty __rawClassName={className} />;}};diff --git a/web/src/features/monitors/components/MonitorsOnboarding.clienttest.tsx b/web/src/features/monitors/components/MonitorsOnboarding.clienttest.tsxindex 98ef54eb2e..fe1be4b296 100644--- a/web/src/features/monitors/components/MonitorsOnboarding.clienttest.tsx+++ b/web/src/features/monitors/components/MonitorsOnboarding.clienttest.tsx@@ -1,7 +1,35 @@import { render, screen } from "@testing-library/react";+import { type Session } from "next-auth";++import type * as EnvModule from "@/src/env.mjs";import { MonitorsOnboarding } from "./MonitorsOnboarding";+const sessionMock = vi.hoisted(() => ({ data: null as Session | null }));+vi.mock("next-auth/react", () => ({+ useSession: () => ({ data: sessionMock.data, status: "authenticated" }),+}));+const cloudRegion = vi.hoisted(() => ({+ value: undefined as string | undefined,+}));+vi.mock("@/src/env.mjs", async (importOriginal) => {+ const actual = await importOriginal<typeof EnvModule>();+ return {+ env: new Proxy(actual.env, {+ get: (target, key) =>+ key === "NEXT_PUBLIC_LANGFUSE_CLOUD_REGION"+ ? cloudRegion.value+ : Reflect.get(target, key),+ }),+ };+});++/** signIn sets the session user's admin flag and the deployment's Cloud region. */+const signIn = ({ admin, region }: { admin: boolean; region?: string }) => {+ sessionMock.data = { expires: "1", user: { admin } } as unknown as Session;+ cloudRegion.value = region;+};+const PROJECT_ID = "proj_test123";/** parsePrefill decodes the `prefill` query param into the same shape the form's serializer produced. */@@ -19,6 +47,11 @@ const parsePrefill = (};describe("MonitorsOnboarding", () => {+ afterEach(() => {+ sessionMock.data = null;+ cloudRegion.value = undefined;+ });+it("renders the splash header, step titles, and four CTAs", () => {render(<MonitorsOnboarding projectId={PROJECT_ID} hasCUDAccess={true} />);@@ -66,6 +99,35 @@ describe("MonitorsOnboarding", () => {}});+ it("hides the PagerDuty CTA from users who are not admins", () => {+ signIn({ admin: false, region: "EU" });+ render(<MonitorsOnboarding projectId={PROJECT_ID} hasCUDAccess={true} />);++ expect(+ screen.queryByRole("link", { name: /Connect PagerDuty/i }),+ ).not.toBeInTheDocument();+ });++ it("hides the PagerDuty CTA from admins on self-hosted instances", () => {+ signIn({ admin: true });+ render(<MonitorsOnboarding projectId={PROJECT_ID} hasCUDAccess={true} />);++ expect(+ screen.queryByRole("link", { name: /Connect PagerDuty/i }),+ ).not.toBeInTheDocument();+ });++ it("shows the PagerDuty CTA to Langfuse Cloud admins", () => {+ signIn({ admin: true, region: "EU" });+ render(<MonitorsOnboarding projectId={PROJECT_ID} hasCUDAccess={true} />);++ const link = screen.getByRole("link", { name: /Connect PagerDuty/i });+ expect(link).not.toHaveTextContent("Internal");+ expect(parsePrefill(link.getAttribute("href")!).actionType).toBe(+ "PAGERDUTY",+ );+ });+it("links the Create Alert CTA to the project's new-alert route", () => {render(<MonitorsOnboarding projectId={PROJECT_ID} hasCUDAccess={true} />);diff --git a/web/src/features/monitors/components/MonitorsOnboarding.tsx b/web/src/features/monitors/components/MonitorsOnboarding.tsxindex 7222683bb6..9a0033f214 100644--- a/web/src/features/monitors/components/MonitorsOnboarding.tsx+++ b/web/src/features/monitors/components/MonitorsOnboarding.tsx@@ -1,11 +1,15 @@import Link from "next/link";import { Github, Plus, Slack, Webhook } from "@langfuse/icons";+import { SiPagerduty } from "@langfuse/icons/si";import { DropdownIndicator } from "@/src/components/design-system/DropdownIndicator/DropdownIndicator";import { ActionButton } from "@/src/components/ActionButton";import { Button } from "@/src/components/ui/button";import { SplashScreen } from "@/src/components/ui/SplashScreen";-import { automationCreateHref } from "@/src/features/automations";+import {+ automationCreateHref,+ useAvailableActionTypes,+} from "@/src/features/automations";import { type ActionTypes } from "@langfuse/shared";/** OnboardingChannel describes one notification-channel CTA shown in step 1 of the splash. */@@ -15,7 +19,7 @@ type OnboardingChannel = {icon: React.ReactNode;};-/** channels enumerates the three notification channels presented to a first-time user. */+/** channels enumerates the notification channels presented to a first-time user. */const channels: OnboardingChannel[] = [{actionType: "SLACK",@@ -32,6 +36,11 @@ const channels: OnboardingChannel[] = [label: "Connect Github Actions",icon: <Github size="base" aria-hidden="true" />,},+ {+ actionType: "PAGERDUTY",+ label: "Connect PagerDuty",+ icon: <SiPagerduty size="base" aria-hidden="true" />,+ },];/** MonitorsOnboarding renders the splash shown on /monitors when the project has no monitors yet. */@@ -42,6 +51,13 @@ export function MonitorsOnboarding({projectId: string;hasCUDAccess: boolean;}) {+ const actionTypes = useAvailableActionTypes(+ channels.map((channel) => channel.actionType),+ );+ const visibleChannels = channels.filter((channel) =>+ actionTypes.includes(channel.actionType),+ );+return (<div className="mx-auto w-full max-w-xl pt-12"><SplashScreen@@ -50,11 +66,10 @@ export function MonitorsOnboarding({steps={[{title: "Choose where alerts should go",- description:- "Send alerts to Slack, Webhooks, or GitHub Actions so your team and your workflows can respond automatically.",+ description: channelsDescription(actionTypes),content: (<div className="flex flex-col gap-2">- {channels.map((channel) => (+ {visibleChannels.map((channel) => (<Buttonkey={channel.actionType}asChild@@ -101,3 +116,11 @@ export function MonitorsOnboarding({</div>);}++/** channelsDescription names the channels the user can connect. */+function channelsDescription(actionTypes: ActionTypes[]): string {+ if (actionTypes.includes("PAGERDUTY")) {+ return "Send alerts to Slack, Webhooks, GitHub Actions, or PagerDuty so your team and your workflows can respond automatically.";+ }+ return "Send alerts to Slack, Webhooks, or GitHub Actions so your team and your workflows can respond automatically.";+}diff --git a/web/src/features/notifications/components/ProjectNotificationChannels.tsx b/web/src/features/notifications/components/ProjectNotificationChannels.tsxindex 624046865a..353b04fe12 100644--- a/web/src/features/notifications/components/ProjectNotificationChannels.tsx+++ b/web/src/features/notifications/components/ProjectNotificationChannels.tsx@@ -25,8 +25,12 @@ import {type ProjectNotificationEventType,} from "@langfuse/shared";-/** Project notifications route to webhooks or Slack; GitHub dispatch is not wired for this event source. */-const PROJECT_NOTIFICATION_ACTION_TYPES: ActionTypes[] = ["WEBHOOK", "SLACK"];+/** Project notifications route to webhooks, Slack or PagerDuty; GitHub dispatch is not wired for this event source. */+const PROJECT_NOTIFICATION_ACTION_TYPES: ActionTypes[] = [+ "WEBHOOK",+ "SLACK",+ "PAGERDUTY",+];const NOTIFIED_EVENT_COPY: Record<ProjectNotificationEventType,diff --git a/web/src/features/notifications/components/ProjectNotificationChannelsList.tsx b/web/src/features/notifications/components/ProjectNotificationChannelsList.tsxindex ed641dda0b..e68a50a616 100644--- a/web/src/features/notifications/components/ProjectNotificationChannelsList.tsx+++ b/web/src/features/notifications/components/ProjectNotificationChannelsList.tsx@@ -1,5 +1,5 @@/* eslint-disable no-nested-ternary */-import { SiSlack } from "@langfuse/icons/si";+import { SiPagerduty, SiSlack } from "@langfuse/icons/si";import { Webhook, Plus, Pencil, Trash2 } from "@langfuse/icons";import { Button } from "@/src/components/ui/button";@@ -17,9 +17,17 @@ const destinationLabel = (automation: AutomationDomain): string => {const config = automation.action.config;return config.type === "WEBHOOK" ? `Webhook ${config.url}` : "Webhook";}+ if (automation.action.type === "PAGERDUTY") return "PagerDuty";return automation.action.type;};+/** ChannelIcon renders the icon of a channel's action type. */+function ChannelIcon({ type }: { type: AutomationDomain["action"]["type"] }) {+ if (type === "SLACK") return <SiSlack size="base" />;+ if (type === "PAGERDUTY") return <SiPagerduty size="base" />;+ return <Webhook size="base" />;+}+export type ProjectNotificationChannelsListProps = {channels: AutomationDomain[] | undefined;isLoading: boolean;@@ -54,11 +62,7 @@ export function ProjectNotificationChannelsList({className="flex items-center justify-between gap-4 rounded-lg border p-4"><div className="flex min-w-0 items-center gap-2">- {channel.action.type === "SLACK" ? (- <SiSlack size="base" />- ) : (- <Webhook size="base" />- )}+ <ChannelIcon type={channel.action.type} /><spanclassName="truncate text-sm font-bold"title={destinationLabel(channel)}diff --git a/web/src/features/pagerduty/components/PagerDutyActionForm.tsx b/web/src/features/pagerduty/components/PagerDutyActionForm.tsxnew file mode 100644index 0000000000..ea8a358d2f--- /dev/null+++ b/web/src/features/pagerduty/components/PagerDutyActionForm.tsx@@ -0,0 +1,111 @@+import { type UseFormReturn } from "react-hook-form";++import { type PagerDutyRegion } from "@langfuse/shared/pagerduty";+import { Form } from "@/src/components/design-system/Form/Form";+import { PasswordInput } from "@/src/components/design-system/PasswordInput/PasswordInput";+import { SelectInput } from "@/src/components/design-system/SelectInput/SelectInput";+import { PagerDutyTestEventButtons } from "./PagerDutyTestEventButtons";++const regionOptions: { value: PagerDutyRegion; label: string }[] = [+ { value: "US", label: "US (events.pagerduty.com)" },+ { value: "EU", label: "EU (events.eu.pagerduty.com)" },+];++/** PagerDutyActionForm collects the routing key and region an automation sends to and can send them a test event. */+export function PagerDutyActionForm({+ form,+ disabled,+ projectId,+ actionId,+}: {+ form: UseFormReturn<PagerDutyFormValues>;+ disabled: boolean;+ projectId: string;+ actionId?: string;+}) {+ const displayRoutingKey = form.watch("pagerDuty.displayRoutingKey");+ const routingKey = form.watch("pagerDuty.routingKey");+ const region = form.watch("pagerDuty.region");++ return (+ <div className="space-y-4">+ <Form.Field+ control={form.control}+ name="pagerDuty.routingKey"+ label="Routing key"+ description={routingKeyDescription(displayRoutingKey)}+ >+ {(field) => (+ <div className="max-w-md">+ <PasswordInput+ id={field.id}+ name={field.name}+ value={field.value}+ onChange={field.onChange}+ onBlur={field.onBlur}+ ref={field.ref}+ disabled={disabled}+ placeholder={displayRoutingKey ?? "32 characters"}+ aria-describedby={field.inputDescribedById}+ aria-invalid={!!field.error}+ error={!!field.error}+ />+ </div>+ )}+ </Form.Field>+ <Form.Field+ control={form.control}+ name="pagerDuty.region"+ label="Region"+ description="The service region of your PagerDuty account."+ >+ {(field) => (+ <div className="max-w-md">+ <SelectInput<PagerDutyRegion>+ id={field.id}+ value={field.value}+ onValueChange={field.onChange}+ disabled={disabled}+ options={regionOptions}+ placeholder="Select a region"+ aria-describedby={field.inputDescribedById}+ />+ </div>+ )}+ </Form.Field>+ <PagerDutyTestEventButtons+ projectId={projectId}+ region={region}+ routingKey={routingKey.trim() || undefined}+ actionId={storedKeyActionId(actionId, displayRoutingKey)}+ disabled={disabled}+ />+ </div>+ );+}++/** routingKeyDescription explains where the key comes from, and that an empty field keeps the stored key. */+function routingKeyDescription(displayRoutingKey: string | undefined): string {+ const source =+ "The integration key of an Events API v2 integration on a PagerDuty service, or an Event Orchestration routing key. Monitor alerts and project notifications trigger incidents; a recovered monitor resolves its incident. Prompt changes are sent as change events, which never page anyone.";+ if (!displayRoutingKey) return source;+ return `${source} Leave empty to keep ${displayRoutingKey}.`;+}++/** storedKeyActionId is the action whose stored key a test event may use, if it has one. */+function storedKeyActionId(+ actionId: string | undefined,+ displayRoutingKey: string | undefined,+): string | undefined {+ if (!displayRoutingKey) return undefined;+ return actionId;+}++/** PagerDutyFormValues is the PagerDuty part of the automation form. */+type PagerDutyFormValues = {+ pagerDuty: {+ routingKey: string;+ displayRoutingKey?: string;+ region: PagerDutyRegion;+ };+};diff --git a/web/src/features/pagerduty/components/PagerDutyTestEventButtons.tsx b/web/src/features/pagerduty/components/PagerDutyTestEventButtons.tsxnew file mode 100644index 0000000000..03b997366f--- /dev/null+++ b/web/src/features/pagerduty/components/PagerDutyTestEventButtons.tsx@@ -0,0 +1,135 @@+import { useState } from "react";+import { CircleCheck, Send } from "@langfuse/icons";++import { type PagerDutyRegion } from "@langfuse/shared/pagerduty";+import { Button } from "@/src/components/design-system/Button/Button";+import { showSuccessToast } from "@/src/features/notifications";+import { useHasProjectAccess } from "@/src/features/rbac";+import { api } from "@/src/utils/api";+import {+ PagerDutyTestEventDialogController,+ type PagerDutyTestEventTarget,+} from "./PagerDutyTestEventDialogController";++/** PagerDutyTestEventButtons triggers a critical test alert, then offers to resolve it. */+export function PagerDutyTestEventButtons({+ projectId,+ region,+ routingKey,+ actionId,+ disabled = false,+}: {+ projectId: string;+ region: PagerDutyRegion;+ routingKey: string | undefined;+ actionId: string | undefined;+ disabled?: boolean;+}) {+ const hasAccess = useHasProjectAccess({+ projectId,+ scope: "automations:CUD",+ });+ const [isTestAlertOpen, setIsTestAlertOpen] = useState(false);+ const isDisabled = disabled || !hasAccess || !(routingKey || actionId);+ const target = { region, routingKey, actionId };++ return (+ <PagerDutyTestEventDialogController+ projectId={projectId}+ onSent={() => setIsTestAlertOpen(true)}+ >+ {({ openDialog }) => (+ <TestEventButton+ projectId={projectId}+ target={target}+ disabled={isDisabled}+ isTestAlertOpen={isTestAlertOpen}+ onSend={() => openDialog(target)}+ onResolved={() => setIsTestAlertOpen(false)}+ />+ )}+ </PagerDutyTestEventDialogController>+ );+}++/** TestEventButton shows the send button, or the resolve button while the test alert is open. */+function TestEventButton({+ projectId,+ target,+ disabled,+ isTestAlertOpen,+ onSend,+ onResolved,+}: {+ projectId: string;+ target: PagerDutyTestEventTarget;+ disabled: boolean;+ isTestAlertOpen: boolean;+ onSend: () => void;+ onResolved: () => void;+}) {+ if (isTestAlertOpen) {+ return (+ <ResolveTestEventButton+ projectId={projectId}+ target={target}+ disabled={disabled}+ onResolved={onResolved}+ />+ );+ }+ return (+ <Button+ type="button"+ text="Send test event"+ variant="secondary"+ size="sm"+ icon={Send}+ disabled={disabled}+ onClick={onSend}+ />+ );+}++/** ResolveTestEventButton resolves the open test alert. */+function ResolveTestEventButton({+ projectId,+ target,+ disabled,+ onResolved,+}: {+ projectId: string;+ target: PagerDutyTestEventTarget;+ disabled: boolean;+ onResolved: () => void;+}) {+ const resolveTestEvent = useResolveTestEvent(onResolved);++ return (+ <Button+ type="button"+ text="Resolve test event"+ variant="secondary"+ size="sm"+ icon={CircleCheck}+ disabled={disabled || resolveTestEvent.isPending}+ onClick={() =>+ resolveTestEvent.mutate({ projectId, ...target, step: "resolve" })+ }+ />+ );+}++/** useResolveTestEvent resolves the test alert and reports it. */+function useResolveTestEvent(onResolved: () => void) {+ return api.automations.sendPagerDutyTestEvent.useMutation({+ onSuccess: () => {+ onResolved();+ showSuccessToast({+ operation: "automations.sendPagerDutyTestEvent",+ title: "Test event resolved",+ description: "PagerDuty accepted the resolve of the test alert.",+ });+ },+ });+}diff --git a/web/src/features/pagerduty/components/PagerDutyTestEventDialogController.tsx b/web/src/features/pagerduty/components/PagerDutyTestEventDialogController.tsxnew file mode 100644index 0000000000..665d9a68e5--- /dev/null+++ b/web/src/features/pagerduty/components/PagerDutyTestEventDialogController.tsx@@ -0,0 +1,55 @@+import { type ReactNode } from "react";++import { type PagerDutyRegion } from "@langfuse/shared/pagerduty";+import { ConfirmationDialogController } from "@/src/components/design-system/ConfirmationDialogController/ConfirmationDialogController";+import { showSuccessToast } from "@/src/features/notifications";+import { api } from "@/src/utils/api";++/** PagerDutyTestEventDialogController warns that a test alert pages whoever is on call, then triggers it. */+export function PagerDutyTestEventDialogController({+ projectId,+ onSent,+ children,+}: {+ projectId: string;+ onSent: () => void;+ children: (control: {+ openDialog: (target: PagerDutyTestEventTarget) => void;+ }) => ReactNode;+}) {+ const sendTestEvent = api.automations.sendPagerDutyTestEvent.useMutation({+ meta: { silentAllErrors: true },+ });++ const handleConfirm = async (target: PagerDutyTestEventTarget) => {+ await sendTestEvent.mutateAsync({ projectId, ...target, step: "trigger" });+ onSent();+ showSuccessToast({+ operation: "automations.sendPagerDutyTestEvent",+ title: "Test event sent",+ description: "PagerDuty accepted the critical test alert.",+ });+ };++ return (+ <ConfirmationDialogController<PagerDutyTestEventTarget>+ title="Send PagerDuty test event"+ text="This triggers a real critical test alert on the PagerDuty service and pages whoever is on call. Resolve it here once it arrives."+ confirmLabel="Send test event"+ variant="default"+ loading={sendTestEvent.isPending}+ error={sendTestEvent.error?.message}+ onAfterDismiss={() => sendTestEvent.reset()}+ onConfirm={handleConfirm}+ >+ {({ openDialog }) => children({ openDialog })}+ </ConfirmationDialogController>+ );+}++/** PagerDutyTestEventTarget is the key a test event goes to: the typed key, else the action's stored key. */+export type PagerDutyTestEventTarget = {+ region: PagerDutyRegion;+ routingKey?: string;+ actionId?: string;+};diff --git a/web/src/features/pagerduty/index.ts b/web/src/features/pagerduty/index.tsnew file mode 100644index 0000000000..b957da4e0a--- /dev/null+++ b/web/src/features/pagerduty/index.ts@@ -0,0 +1 @@+export { PagerDutyActionForm } from "@/src/features/pagerduty/components/PagerDutyActionForm";diff --git a/web/src/features/pagerduty/server/index.ts b/web/src/features/pagerduty/server/index.tsnew file mode 100644index 0000000000..715b232bd2--- /dev/null+++ b/web/src/features/pagerduty/server/index.ts@@ -0,0 +1,5 @@+export {+ processPagerDutyActionConfig,+ sendPagerDutyTestEvent,+ throwIfPagerDutyUnavailable,+} from "@/src/features/pagerduty/server/pagerdutyHelpers";diff --git a/web/src/features/pagerduty/server/pagerdutyHelpers.ts b/web/src/features/pagerduty/server/pagerdutyHelpers.tsnew file mode 100644index 0000000000..9c232eae77--- /dev/null+++ b/web/src/features/pagerduty/server/pagerdutyHelpers.ts@@ -0,0 +1,181 @@+import { type Session } from "next-auth";+import {+ BaseError,+ ForbiddenError,+ InvalidRequestError,+ isPagerDutyActionConfig,+ type ActionCreate,+ type PagerDutyActionConfigWithSecrets,+} from "@langfuse/shared";+import { decrypt, encrypt } from "@langfuse/shared/encryption";+import { prisma } from "@langfuse/shared/src/db";+import {+ isRetryablePagerDutyError,+ maskPagerDutyRoutingKey,+ pagerDutyClient,+ type PagerDutyAlertEvent,+ type PagerDutyRegion,+ type PagerDutyTestEventStep,+} from "@langfuse/shared/pagerduty/server";+import { getProductBaseUrl, logger } from "@langfuse/shared/src/server";+import { env } from "@/src/env.mjs";++/** throwIfPagerDutyUnavailable limits PagerDuty actions to Langfuse Cloud admins. */+export function throwIfPagerDutyUnavailable(session: Session): void {+ if (env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION && session.user?.admin === true)+ return;+ throw new ForbiddenError("PagerDuty actions are not available yet");+}++/** processPagerDutyActionConfig encrypts a new routing key, or keeps the stored one when the update omits it. */+export async function processPagerDutyActionConfig({+ actionConfig,+ actionId,+ projectId,+}: {+ actionConfig: ActionCreate;+ actionId?: string;+ projectId: string;+}): Promise<PagerDutyActionConfigWithSecrets> {+ if (actionConfig.type !== "PAGERDUTY") {+ throw new InvalidRequestError("Invalid PagerDuty action configuration");+ }++ if (actionConfig.routingKey) {+ return {+ type: "PAGERDUTY",+ routingKey: encrypt(actionConfig.routingKey),+ displayRoutingKey: maskPagerDutyRoutingKey(actionConfig.routingKey),+ region: actionConfig.region,+ };+ }++ const existing = await getStoredPagerDutyConfig({ actionId, projectId });+ if (!existing) {+ throw new InvalidRequestError("PagerDuty routing key is required");+ }+ return { ...existing, region: actionConfig.region };+}++/** getStoredPagerDutyConfig loads the PagerDuty config of an existing action, or null when there is none. */+async function getStoredPagerDutyConfig({+ actionId,+ projectId,+}: {+ actionId?: string;+ projectId: string;+}): Promise<PagerDutyActionConfigWithSecrets | null> {+ if (!actionId) return null;+ const action = await prisma.action.findFirst({+ where: { id: actionId, projectId },+ select: { config: true },+ });+ const config = action?.config;+ if (!isPagerDutyActionConfig(config)) return null;+ return config;+}++/** sendPagerDutyTestEvent triggers or resolves the project's test alert, which is keyed by project. */+export async function sendPagerDutyTestEvent({+ projectId,+ region,+ routingKey,+ actionId,+ step,+}: {+ projectId: string;+ region: PagerDutyRegion;+ routingKey?: string;+ actionId?: string;+ step: PagerDutyTestEventStep;+}): Promise<void> {+ const key = await resolveTestRoutingKey({ projectId, routingKey, actionId });+ const event = await buildTestEvent({ projectId, routingKey: key, step });+ try {+ await pagerDutyClient.alert({ region, event });+ } catch (error) {+ throw toTestEventError(error, projectId);+ }+}++/** resolveTestRoutingKey prefers the typed key and falls back to the stored key of the action. */+async function resolveTestRoutingKey({+ projectId,+ routingKey,+ actionId,+}: {+ projectId: string;+ routingKey?: string;+ actionId?: string;+}): Promise<string> {+ if (routingKey) return routingKey;+ const existing = await getStoredPagerDutyConfig({ actionId, projectId });+ if (!existing) {+ throw new InvalidRequestError(+ "Enter a PagerDuty routing key to send a test event",+ );+ }+ return decrypt(existing.routingKey);+}++/** buildTestEvent resolves the test alert, or triggers it at critical severity. */+async function buildTestEvent({+ projectId,+ routingKey,+ step,+}: {+ projectId: string;+ routingKey: string;+ step: PagerDutyTestEventStep;+}): Promise<PagerDutyAlertEvent> {+ const dedupKey = `langfuse-${projectId}-test`;+ if (step === "resolve") {+ return {+ routing_key: routingKey,+ event_action: "resolve",+ dedup_key: dedupKey,+ };+ }++ const project = await prisma.project.findUniqueOrThrow({+ where: { id: projectId },+ select: { name: true },+ });+ const projectUrl = new URL(+ `project/${projectId}`,+ getProductBaseUrl(),+ ).toString();+ return {+ routing_key: routingKey,+ event_action: "trigger",+ dedup_key: dedupKey,+ client: "Langfuse",+ client_url: projectUrl,+ links: [{ href: projectUrl, text: "Open Langfuse project" }],+ payload: {+ summary: `[Langfuse] Test event from project ${project.name}`,+ source: `langfuse/${project.name}`,+ severity: "critical",+ timestamp: new Date().toISOString(),+ class: "test",+ custom_details: {+ message:+ "This is a test event from Langfuse to verify your PagerDuty automation. Resolve it from the automation form or in PagerDuty.",+ },+ },+ };+}++/** toTestEventError keeps rate limits and outages as they are and turns any other PagerDuty rejection into an InvalidRequestError. */+function toTestEventError(error: unknown, projectId: string): unknown {+ if (!(error instanceof BaseError)) return error;+ logger.warn("Failed to send PagerDuty test event", {+ projectId,+ status: error.httpCode,+ error: error.message,+ });+ if (isRetryablePagerDutyError(error)) return error;+ return new InvalidRequestError(+ `PagerDuty did not accept the test event. Check the routing key and region. (${error.message})`,+ );+}diff --git a/web/src/features/slack/components/ChannelSelector.tsx b/web/src/features/slack/components/ChannelSelector.tsxindex 040af519a3..39d4f54ef6 100644--- a/web/src/features/slack/components/ChannelSelector.tsx+++ b/web/src/features/slack/components/ChannelSelector.tsx@@ -41,6 +41,10 @@ export const ChannelSelector: React.FC<ChannelSelectorProps> = ({projectId,scope: "automations:CUD",});+ const canManageIntegration = useHasProjectAccess({+ projectId,+ scope: "integrations:CRUD",+ });const { channelsData, isLoadingChannels, error, refetchChannels } =useSlackChannels(projectId, canListChannels);@@ -70,7 +74,7 @@ export const ChannelSelector: React.FC<ChannelSelectorProps> = ({hasError={!!error}isRefreshing={isRefreshing}onRefresh={handleRefresh}- onReauthenticate={handleReauthenticate}+ onReauthenticate={canManageIntegration ? handleReauthenticate : undefined}/>);};diff --git a/web/src/features/slack/components/ChannelSelectorView/ChannelSelectorView.tsx b/web/src/features/slack/components/ChannelSelectorView/ChannelSelectorView.tsxindex e33ca953ce..6c34400295 100644--- a/web/src/features/slack/components/ChannelSelectorView/ChannelSelectorView.tsx+++ b/web/src/features/slack/components/ChannelSelectorView/ChannelSelectorView.tsx@@ -38,8 +38,8 @@ export interface ChannelSelectorViewProps {/** Callback when a channel is selected */onChannelSelect: (channel: SlackChannel) => void;onRefresh: () => void;- /** Reconnects Slack to grant missing scopes */- onReauthenticate: () => void;+ /** Reconnects Slack to grant missing scopes; omitted for users who may not manage the integration */+ onReauthenticate?: () => void;/** Whether the component is disabled */disabled?: boolean;/** Placeholder text for the selector */@@ -296,15 +296,21 @@ export const ChannelSelectorView: React.FC<ChannelSelectorViewProps> = ({<Alert icon={AlertTriangle} variant="warning" size="sm"><Alert.Title>Private channels are not visible</Alert.Title><Alert.Description>- To access private channels,{" "}- <button- type="button"- className="font-bold underline"- onClick={onReauthenticate}- >- re-authenticate your Slack integration- </button>{" "}- to grant the required permissions.+ {onReauthenticate ? (+ <>+ To access private channels,{" "}+ <button+ type="button"+ className="font-bold underline"+ onClick={onReauthenticate}+ >+ re-authenticate your Slack integration+ </button>{" "}+ to grant the required permissions.+ </>+ ) : (+ "To access private channels, ask a project admin to re-authenticate the Slack integration."+ )}</Alert.Description></Alert>)}diff --git a/web/src/features/slack/components/SlackConnectionCard.tsx b/web/src/features/slack/components/SlackConnectionCard.tsxindex 5ffebece8b..c6c2e0bd8b 100644--- a/web/src/features/slack/components/SlackConnectionCard.tsx+++ b/web/src/features/slack/components/SlackConnectionCard.tsx@@ -13,6 +13,7 @@ import { Alert } from "@/src/components/design-system/Alert/Alert";import { api } from "@/src/utils/api";import { SlackConnectButton } from "@/src/features/slack/components/SlackConnectButton";import { SlackDisconnectDialogController } from "@/src/features/slack/components/SlackDisconnectDialogController";+import { useHasProjectAccess } from "@/src/features/rbac";/*** Props for the SlackConnectionCard component@@ -50,6 +51,13 @@ export const SlackConnectionCard: React.FC<SlackConnectionCardProps> = ({onConnectionChange,showConnectButton = true,}) => {+ // Connecting and disconnecting a workspace is project integration config,+ // separate from managing the automations that use it.+ const canManageIntegration = useHasProjectAccess({+ projectId,+ scope: "integrations:CRUD",+ });+// Get Slack integration statusconst {data: integrationStatus,@@ -137,10 +145,16 @@ export const SlackConnectionCard: React.FC<SlackConnectionCardProps> = ({your automations.</p>+ {!canManageIntegration && (+ <p className="text-muted-foreground text-sm">+ Ask a project admin to connect Slack.+ </p>+ )}+{showConnectButton && (<SlackConnectButtonprojectId={projectId}- disabled={disabled}+ disabled={disabled || !canManageIntegration}onSuccess={() => handleConnectionChange(true)}onError={(error: Error) => {console.error("Slack connection error:", error);@@ -213,7 +227,7 @@ export const SlackConnectionCard: React.FC<SlackConnectionCardProps> = ({variant="destructive"size="sm"onClick={openDialog}- disabled={disabled || isPending}+ disabled={disabled || isPending || !canManageIntegration}/>)}</SlackDisconnectDialogController>diff --git a/web/src/features/slack/server/router.ts b/web/src/features/slack/server/router.tsindex f751b35905..ebf94e1bcb 100644--- a/web/src/features/slack/server/router.ts+++ b/web/src/features/slack/server/router.ts@@ -194,7 +194,7 @@ export const slackRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: input.projectId,- scope: "automations:CUD",+ scope: "integrations:CRUD",});const integration = await ctx.prisma.slackIntegration.findUnique({diff --git a/web/src/pages/api/public/slack/install/index.ts b/web/src/pages/api/public/slack/install/index.tsindex 874bcba2f1..9731570361 100644--- a/web/src/pages/api/public/slack/install/index.ts+++ b/web/src/pages/api/public/slack/install/index.ts@@ -29,11 +29,11 @@ export default async function handler(return res.status(401).json({ error: "Authentication required" });}- // Verify user has permission to modify automations for this project+ // Connecting a workspace is project integration configconst hasAccess = hasProjectAccess({session,projectId,- scope: "automations:CUD",+ scope: "integrations:CRUD",});if (!hasAccess) {diff --git a/worker/src/__tests__/monitorProcessor.test.ts b/worker/src/__tests__/monitorProcessor.test.tsindex 0e92996c38..54e7db27fc 100644--- a/worker/src/__tests__/monitorProcessor.test.ts+++ b/worker/src/__tests__/monitorProcessor.test.ts@@ -1698,6 +1698,50 @@ describe("MonitorProcessor.process evaluation offset", () => {// The cadence-boundary stamp stays unshifted — alerts say "fired at runAt".expect(sent.payload.timestamp.toISOString()).toBe(runAt.toISOString());});++ it("carries the monitor config, evaluated value and alert time on the alert", async () => {+ const monitorId = `m_cfg_${v4()}`;+ await seedMonitor(projectId, {+ id: monitorId,+ severity: MonitorSeveritySchema.enum.OK,+ alertThreshold: 100,+ warningThreshold: 50,+ lastPublishedAt: runAt,+ triggerIds: ["trig_cfg"],+ });+ const publish = vi.fn<MonitorPublisher>(async () => {});+ const getTriggers: GetTriggerConfigurations = async () =>+ [+ {+ id: "trig_cfg",+ filter: matchAnyAlertTrigger.filter,+ eventActions: [],+ automations: [{ id: "auto_cfg", actionId: "act_cfg" }],+ },+ ] as unknown as Awaited<ReturnType<GetTriggerConfigurations>>;+ const processor = new MonitorProcessor(+ prisma,+ publish,+ async () => [{ count_count: 150 }],+ getTriggers,+ );++ await processor.process(makeEvent(projectId, [monitorId]), justAfterRunAt);++ expect(publish).toHaveBeenCalledTimes(1);+ const sent = publish.mock.calls[0][0].payload;+ if (sent.type !== "monitor-alert") throw new Error("unexpected envelope");+ expect(sent.payload).toMatchObject({+ severity: MonitorSeveritySchema.enum.ALERT,+ monitorName: `Test ${monitorId}`,+ metric: { measure: "count", aggregation: "count" },+ metricValue: 150,+ thresholdOperator: MonitorThresholdOperatorSchema.enum.GT,+ alertThreshold: 100,+ warningThreshold: 50,+ alertedAt: justAfterRunAt,+ });+ });});describe("MonitorProcessor.process wire deserialization", () => {diff --git a/worker/src/__tests__/slack-processor.test.ts b/worker/src/__tests__/slack-processor.test.tsindex 6c92aa8c5d..aefb3402ba 100644--- a/worker/src/__tests__/slack-processor.test.ts+++ b/worker/src/__tests__/slack-processor.test.ts@@ -39,6 +39,7 @@ describe("Slack Processor", () => {let automationId: string;let promptId: string;let executionId: string;+ let monitorId: string;let mockSlackService: any;beforeAll(async () => {@@ -93,6 +94,24 @@ describe("Slack Processor", () => {},});+ // monitor-alert jobs are only delivered while their monitor exists+ ({ id: monitorId } = await prisma.monitor.create({+ data: {+ projectId,+ name: "Test monitor",+ view: "OBSERVATIONS",+ filters: [],+ metric: { measure: "count", aggregation: "count" },+ windowMs: 5n * 60_000n,+ cadenceMs: 60_000n,+ thresholdOperator: "GT",+ alertThreshold: 100,+ noData: { mode: "SHOW_NO_DATA" },+ renotify: { mode: "OFF" },+ schedulerBatchId: 0n,+ },+ }));+// Create test promptpromptId = v4();await prisma.prompt.create({@@ -219,6 +238,42 @@ describe("Slack Processor", () => {expect(execution?.finishedAt).toBeDefined();});+ it("delivers prompt changes whose config is not a JSON object", async () => {+ const fullPrompt = await prisma.prompt.findUniqueOrThrow({+ where: { id: promptId },+ });+ const slackInput: WebhookInput = {+ projectId,+ automationId,+ executionId,+ payload: {+ prompt: { ...(fullPrompt as PromptDomain), config: [] },+ action: "created",+ type: "prompt-version",+ },+ };+ await prisma.automationExecution.create({+ data: {+ id: executionId,+ projectId,+ triggerId,+ automationId,+ actionId,+ status: ActionExecutionStatus.PENDING,+ sourceId: slackInput.executionId,+ input: {},+ },+ });++ await executeWebhook(slackInput);++ expect(mockSlackService.sendMessage).toHaveBeenCalled();+ const execution = await prisma.automationExecution.findUniqueOrThrow({+ where: { id: executionId },+ });+ expect(execution.status).toBe(ActionExecutionStatus.COMPLETED);+ });+it("should handle missing automation gracefully", async () => {const nonExistentAutomationId = v4();@@ -535,7 +590,7 @@ describe("Slack Processor", () => {type: "monitor-alert",apiVersion: "v1",payload: {- monitorId: "mon_01",+ monitorId,projectId,severity: "ALERT",permalink:@@ -596,7 +651,7 @@ describe("Slack Processor", () => {type: "monitor-alert",apiVersion: "v1",payload: {- monitorId: "mon_01",+ monitorId,projectId,severity: "ALERT",permalink:@@ -647,7 +702,7 @@ describe("Slack Processor", () => {type: "monitor-alert",apiVersion: "v1",payload: {- monitorId: "mon_01",+ monitorId,projectId,severity: "ALERT",permalink:diff --git a/worker/src/__tests__/webhooks.test.ts b/worker/src/__tests__/webhooks.test.tsindex b5b1841fb2..880c518765 100644--- a/worker/src/__tests__/webhooks.test.ts+++ b/worker/src/__tests__/webhooks.test.ts@@ -20,6 +20,7 @@ import {import {WebhookInput,createOrgProjectAndApiKey,+ getActionById,getActionByIdWithSecrets,logger,redis,@@ -92,9 +93,26 @@ class WebhookTestServer {headers: Record<string, string>;body: string;}> = [];+ pagerDutyStatus = 202;constructor() {+ const recordPagerDutyEvent = async ({ request }: { request: Request }) => {+ this.receivedRequests.push({+ url: request.url,+ method: request.method,+ headers: Object.fromEntries(request.headers.entries()),+ body: JSON.stringify(await request.json()),+ });+ return HttpResponse.json(+ { status: "success", message: "Event processed" },+ { status: this.pagerDutyStatus },+ );+ };+this.server = setupServer(+ http.post("https://events.pagerduty.com/*", recordPagerDutyEvent),+ http.post("https://events.eu.pagerduty.com/*", recordPagerDutyEvent),+// Default success responsehttp.post("https://webhook.example.com/*", async ({ request }) => {this.receivedRequests.push({@@ -144,6 +162,7 @@ class WebhookTestServer {reset() {this.receivedRequests = [];+ this.pagerDutyStatus = 202;this.server.resetHandlers();}@@ -161,6 +180,28 @@ class WebhookTestServer {}const webhookServer = new WebhookTestServer();++/** createMonitor writes a minimal Monitor row so monitor-alert jobs pass the stale-alert guard. */+async function createMonitor(projectId: string): Promise<string> {+ const monitor = await prisma.monitor.create({+ data: {+ projectId,+ name: "Test monitor",+ view: "OBSERVATIONS",+ filters: [],+ metric: { measure: "count", aggregation: "count" },+ windowMs: 5n * 60_000n,+ cadenceMs: 60_000n,+ thresholdOperator: "GT",+ alertThreshold: 100,+ noData: { mode: "SHOW_NO_DATA" },+ renotify: { mode: "OFF" },+ schedulerBatchId: 0n,+ severity: "ALERT",+ },+ });+ return monitor.id;+}const GITHUB_REPOSITORY_DISPATCH_TRUNCATION_MARKER ="[TRUNCATED: GitHub repository_dispatch payload exceeded size limit]";const GITHUB_REPOSITORY_DISPATCH_MAX_PAYLOAD_BYTES = 64 * 1024;@@ -172,6 +213,7 @@ describe("Webhook Integration Tests", () => {let automationId: string;let promptId: string;let executionId: string;+ let monitorId: string;beforeAll(() => {webhookServer.setup();@@ -244,6 +286,7 @@ describe("Webhook Integration Tests", () => {},});executionId = v4();+ monitorId = await createMonitor(projectId);});afterEach(() => {@@ -844,7 +887,7 @@ describe("Webhook Integration Tests", () => {type: "monitor-alert",apiVersion: "v1",payload: {- monitorId: v4(),+ monitorId,projectId,severity: "ALERT",permalink: `https://cloud.langfuse.com/project/${projectId}/monitors/mon`,@@ -1024,7 +1067,7 @@ describe("Webhook Integration Tests", () => {type: "monitor-alert",apiVersion: "v1",payload: {- monitorId: v4(),+ monitorId,projectId,severity: "ALERT",permalink: `https://cloud.langfuse.com/project/${projectId}/monitors/mon`,@@ -2193,7 +2236,7 @@ describe("Webhook Integration Tests", () => {type: "monitor-alert",apiVersion: "v1",payload: {- monitorId: v4(),+ monitorId,projectId,severity: "ALERT",permalink: `https://cloud.langfuse.com/project/${projectId}/monitors/mon`,@@ -2236,6 +2279,479 @@ describe("Webhook Integration Tests", () => {});});});++ describe("stale monitor alerts", () => {+ const alertedAt = new Date("2026-05-27T14:00:00.000Z");++ const buildMonitorAlertInput = (alertAlertedAt?: Date) =>+ ({+ projectId,+ automationId,+ executionId: v4(),+ payload: {+ id: v4(),+ timestamp: new Date(),+ type: "monitor-alert",+ apiVersion: "v1",+ payload: {+ monitorId,+ projectId,+ severity: "ALERT",+ timestamp: new Date(),+ fromTimestamp: new Date(Date.now() - 5 * 60_000),+ toTimestamp: new Date(),+ message: { title: "High error rate", body: "errors > 100" },+ view: "observations",+ filters: [],+ window: "5m",+ alertedAt: alertAlertedAt,+ },+ },+ }) satisfies WebhookInput;++ const useErrorEndpoint = async () => {+ const action = await prisma.action.findUniqueOrThrow({+ where: { id: actionId },+ });+ await prisma.action.update({+ where: { id: actionId },+ data: {+ config: {+ ...(action.config as WebhookActionConfigWithSecrets),+ url: "https://webhook-error.example.com/test",+ },+ },+ });+ };++ it("drops the alert without counting a failure once the monitor emitted a newer alert", async () => {+ await prisma.monitor.update({+ where: { id: monitorId },+ data: { alertedAt: new Date(alertedAt.getTime() + 1) },+ });+ await useErrorEndpoint();+ const failureKey = `automation-failures:${projectId}:${automationId}`;+ await redis!.del(failureKey);++ await executeWebhook(buildMonitorAlertInput(alertedAt), {+ skipValidation: true,+ });++ expect(webhookServer.getReceivedRequests()).toHaveLength(0);+ expect(await redis!.get(failureKey)).toBeNull();+ });++ it("drops the alert when its monitor was deleted", async () => {+ await prisma.monitor.delete({ where: { id: monitorId } });++ await executeWebhook(buildMonitorAlertInput(alertedAt), {+ skipValidation: true,+ });++ expect(webhookServer.getReceivedRequests()).toHaveLength(0);+ });++ it("drops the alert for every action type, not only webhooks", async () => {+ await prisma.monitor.update({+ where: { id: monitorId },+ data: { alertedAt: new Date(alertedAt.getTime() + 1) },+ });+ await prisma.action.update({+ where: { id: actionId },+ data: {+ type: "GITHUB_DISPATCH",+ config: {+ type: "GITHUB_DISPATCH",+ url: "https://webhook.example.com/dispatches",+ eventType: "monitor-alert",+ githubToken: encrypt("ghp_test_token"),+ displayGitHubToken: "ghp_...n",+ },+ },+ });++ await executeWebhook(buildMonitorAlertInput(alertedAt), {+ skipValidation: true,+ });++ expect(webhookServer.getReceivedRequests()).toHaveLength(0);+ });++ it.each([+ // The processor publishes before it saves alertedAt.+ ["before the monitor saved it", new Date(0)],+ ["after the monitor saved it", alertedAt],+ ])("delivers the alert %s", async (_, monitorAlertedAt) => {+ await prisma.monitor.update({+ where: { id: monitorId },+ data: { alertedAt: monitorAlertedAt },+ });++ await executeWebhook(buildMonitorAlertInput(alertedAt), {+ skipValidation: true,+ });++ expect(webhookServer.getReceivedRequests()).toHaveLength(1);+ });++ it("delivers the alert when the monitor changed severity without emitting an alert", async () => {+ await prisma.monitor.update({+ where: { id: monitorId },+ data: {+ severityChangedAt: new Date(alertedAt.getTime() + 1),+ alertedAt,+ },+ });++ await executeWebhook(buildMonitorAlertInput(alertedAt), {+ skipValidation: true,+ });++ expect(webhookServer.getReceivedRequests()).toHaveLength(1);+ });++ it("delivers alerts queued without alertedAt", async () => {+ await prisma.monitor.update({+ where: { id: monitorId },+ data: { alertedAt: new Date() },+ });++ await executeWebhook(buildMonitorAlertInput(), {+ skipValidation: true,+ });++ expect(webhookServer.getReceivedRequests()).toHaveLength(1);+ });+ });++ describe("PagerDuty action", () => {+ const routingKey = "R0abcdefghijklmnopqrstuvwxyz9xYz";++ const setPagerDutyConfig = (region: "US" | "EU") =>+ prisma.action.update({+ where: { id: actionId },+ data: {+ type: "PAGERDUTY",+ config: {+ type: "PAGERDUTY",+ routingKey: encrypt(routingKey),+ displayRoutingKey: "R0ab...9xYz",+ region,+ },+ },+ });++ const buildMonitorInput = (+ severity: "ALERT" | "WARNING" | "OK" | "NO_DATA",+ ): WebhookInput => ({+ projectId,+ automationId,+ executionId: v4(),+ payload: {+ id: v4(),+ timestamp: new Date(),+ type: "monitor-alert",+ apiVersion: "v1",+ payload: {+ monitorId,+ projectId,+ severity,+ permalink: `https://cloud.langfuse.com/project/${projectId}/alerts/${monitorId}`,+ timestamp: new Date("2026-05-27T14:32:00.000Z"),+ fromTimestamp: new Date("2026-05-27T14:00:00.000Z"),+ toTimestamp: new Date("2026-05-27T14:30:00.000Z"),+ message: { title: "Hallucination rate breached", body: "0.42 > 0.3" },+ view: "observations",+ filters: [+ {+ column: "providedModelName",+ type: "stringOptions",+ operator: "any of",+ value: ["gpt-4o"],+ },+ ],+ window: "30m",+ monitorName: "Hallucination rate",+ metric: { measure: "latency", aggregation: "p95" },+ metricValue: 0.42,+ thresholdOperator: "GT",+ alertThreshold: 0.3,+ warningThreshold: 0.2,+ },+ },+ });++ const pagerDutyRequests = () =>+ webhookServer.getReceivedRequests().map((request) => ({+ url: request.url,+ body: JSON.parse(request.body),+ }));++ const lastPagerDutyRequest = () => pagerDutyRequests().at(-1)!;++ beforeEach(async () => {+ await setPagerDutyConfig("US");+ await redis!.del(`automation-failures:${projectId}:${automationId}`);+ });++ it("triggers a critical alert keyed by the monitor with LLMOps details", async () => {+ await executeWebhook(buildMonitorInput("ALERT"));++ const [{ url, body }] = pagerDutyRequests();+ const project = await prisma.project.findUniqueOrThrow({+ where: { id: projectId },+ });+ expect(url).toBe("https://events.pagerduty.com/v2/enqueue");+ expect(body).toMatchObject({+ routing_key: routingKey,+ event_action: "trigger",+ dedup_key: `langfuse-${projectId}-${monitorId}`,+ client: "Langfuse",+ links: [+ {+ href: `https://cloud.langfuse.com/project/${projectId}/alerts/${monitorId}`,+ },+ ],+ payload: {+ summary: "Hallucination rate breached",+ source: `langfuse/${project.name}`,+ severity: "critical",+ timestamp: "2026-05-27T14:32:00.000Z",+ class: "monitor-alert",+ component: "observations",+ custom_details: {+ llmops: {+ model_name: "gpt-4o",+ project_name: project.name,+ monitor_name: "Hallucination rate",+ monitor_id: monitorId,+ monitor_alert_start: "2026-05-27T14:00:00.000Z",+ monitor_alert_end: "2026-05-27T14:30:00.000Z",+ metric_value: 0.42,+ threshold_value: 0.3,+ },+ langfuse: { window: "30m" },+ },+ },+ });+ });++ it.each([+ ["WARNING", "warning"],+ ["NO_DATA", "error"],+ ] as const)(+ "maps %s to PagerDuty severity %s",+ async (severity, expected) => {+ await executeWebhook(buildMonitorInput(severity));++ expect(pagerDutyRequests()[0].body.payload.severity).toBe(expected);+ },+ );++ it("re-triggers the monitor's alert under the same key on a severity change", async () => {+ await executeWebhook(buildMonitorInput("WARNING"));+ await executeWebhook(buildMonitorInput("ALERT"));++ expect(+ pagerDutyRequests().map(({ body }) => [+ body.event_action,+ body.dedup_key,+ body.payload.severity,+ ]),+ ).toEqual([+ ["trigger", `langfuse-${projectId}-${monitorId}`, "warning"],+ ["trigger", `langfuse-${projectId}-${monitorId}`, "critical"],+ ]);+ });++ it("resolves the monitor's alert on recovery", async () => {+ await executeWebhook(buildMonitorInput("OK"));++ expect(pagerDutyRequests().map(({ body }) => body)).toEqual([+ {+ routing_key: routingKey,+ event_action: "resolve",+ dedup_key: `langfuse-${projectId}-${monitorId}`,+ },+ ]);+ });++ it("sends to the EU host for EU routing keys", async () => {+ await setPagerDutyConfig("EU");++ await executeWebhook(buildMonitorInput("ALERT"));++ expect(pagerDutyRequests()[0].url).toBe(+ "https://events.eu.pagerduty.com/v2/enqueue",+ );+ });++ it("sends prompt changes as change events and completes the execution", async () => {+ const fullPrompt = await prisma.prompt.findUniqueOrThrow({+ where: { id: promptId },+ });+ const input: WebhookInput = {+ projectId,+ automationId,+ executionId,+ payload: {+ prompt: PromptDomainSchema.parse(fullPrompt),+ action: "created",+ type: "prompt-version",+ },+ };+ await prisma.automationExecution.create({+ data: {+ id: executionId,+ projectId,+ triggerId,+ automationId,+ actionId,+ status: ActionExecutionStatus.PENDING,+ sourceId: executionId,+ input: {},+ },+ });++ await executeWebhook(input);++ const { url, body } = lastPagerDutyRequest();+ expect(url).toBe("https://events.pagerduty.com/v2/change/enqueue");+ expect(body).toMatchObject({+ routing_key: routingKey,+ payload: {+ summary: 'Langfuse prompt "test-prompt" version 1 created',+ custom_details: { prompt_name: "test-prompt", prompt_version: 1 },+ },+ });+ expect(body).not.toHaveProperty("event_action");+ const execution = await prisma.automationExecution.findUniqueOrThrow({+ where: { id: executionId },+ });+ expect(execution.status).toBe(ActionExecutionStatus.COMPLETED);+ });++ it("always triggers project notifications keyed by event type and resource", async () => {+ await prisma.automationExecution.create({+ data: {+ id: executionId,+ projectId,+ triggerId,+ automationId,+ actionId,+ status: ActionExecutionStatus.PENDING,+ sourceId: executionId,+ input: {},+ },+ });++ await executeWebhook({+ projectId,+ automationId,+ executionId,+ payload: {+ id: executionId,+ timestamp: new Date(),+ type: "project-notification",+ apiVersion: "v1",+ event: {+ eventType: "blob-export-failed",+ severity: "ALERT",+ projectId,+ projectName: "Test Project",+ resourceId: "res_1",+ resourceName: "my-bucket",+ message: "Blob storage export failed.",+ },+ },+ });++ expect(lastPagerDutyRequest().body).toMatchObject({+ event_action: "trigger",+ dedup_key: `langfuse-${projectId}-blob-export-failed-res_1`,+ payload: { severity: "error", class: "blob-export-failed" },+ });+ });++ it.each([429, 500])(+ "retries a %i in-process, then counts a failure once retries run out",+ async (status) => {+ webhookServer.pagerDutyStatus = status;++ await executeWebhook(buildMonitorInput("ALERT"));++ expect(webhookServer.getReceivedRequests()).toHaveLength(4);+ expect(+ await redis!.get(`automation-failures:${projectId}:${automationId}`),+ ).toBe("1");+ },+ );++ it("counts a rejected event as a failure without retrying", async () => {+ webhookServer.pagerDutyStatus = 400;++ await executeWebhook(buildMonitorInput("ALERT"));++ expect(+ await redis!.get(`automation-failures:${projectId}:${automationId}`),+ ).toBe("1");+ });++ it("records PagerDuty's status on a rejected prompt change", async () => {+ webhookServer.pagerDutyStatus = 400;+ const fullPrompt = await prisma.prompt.findUniqueOrThrow({+ where: { id: promptId },+ });+ await prisma.automationExecution.create({+ data: {+ id: executionId,+ projectId,+ triggerId,+ automationId,+ actionId,+ status: ActionExecutionStatus.PENDING,+ sourceId: executionId,+ input: {},+ },+ });++ await executeWebhook({+ projectId,+ automationId,+ executionId,+ payload: {+ prompt: PromptDomainSchema.parse(fullPrompt),+ action: "created",+ type: "prompt-version",+ },+ });++ const execution = await prisma.automationExecution.findUniqueOrThrow({+ where: { id: executionId },+ });+ expect(execution).toMatchObject({+ status: ActionExecutionStatus.ERROR,+ output: { httpStatus: 400 },+ });+ });++ it("returns the routing key only on the execution read path", async () => {+ const withSecrets = await getActionByIdWithSecrets({+ projectId,+ actionId,+ });+ const safe = await getActionById({ projectId, actionId });++ const config = withSecrets?.config as { routingKey: string };+ expect(decrypt(config.routingKey)).toBe(routingKey);+ expect(safe?.config).toEqual({+ type: "PAGERDUTY",+ displayRoutingKey: "R0ab...9xYz",+ region: "US",+ });+ expect(JSON.stringify(safe)).not.toContain(routingKey);+ });+ });});describe("getActionByIdWithSecrets legacy headers", () => {diff --git a/worker/src/features/pagerduty/buildPagerDutyEvent.ts b/worker/src/features/pagerduty/buildPagerDutyEvent.tsnew file mode 100644index 0000000000..4289203ad3--- /dev/null+++ b/worker/src/features/pagerduty/buildPagerDutyEvent.ts@@ -0,0 +1,336 @@+import { buildPromptUrl, getProductBaseUrl } from "@langfuse/shared/src/server";+import {+ truncatePagerDutySummary,+ type PagerDutyAlertEvent,+ type PagerDutyChangeEvent,+ type PagerDutyLink,+ type PagerDutySeverity,+} from "@langfuse/shared/pagerduty/server";+import { type MonitorAlert } from "@langfuse/shared/monitors";++const client = "Langfuse";++const monitorSeverities: Partial<+ Record<MonitorAlert["severity"], PagerDutySeverity>+> = {+ ALERT: "critical",+ WARNING: "warning",+ NO_DATA: "error",+};++const notificationSeverities: Record<+ ProjectNotificationEvent["severity"],+ PagerDutySeverity+> = {+ ALERT: "error",+ WARNING: "warning",+ INFO: "info",+};++const modelFilterColumns = new Set([+ "providedModelName",+ "observationModelName",+]);++/** buildPagerDutyEvent maps a validated webhook envelope to its PagerDuty event, or undefined when the alert has nothing to send. */+export function buildPagerDutyEvent(args: {+ envelope: PagerDutyEnvelope;+ projectId: string;+ projectName: string;+ routingKey: string;+}): PagerDutyEvent | undefined {+ const context = {+ projectId: args.projectId,+ projectName: args.projectName,+ routingKey: args.routingKey,+ };+ switch (args.envelope.type) {+ case "monitor-alert":+ return buildMonitorAlertEvent(args.envelope.payload, context);+ case "project-notification":+ return buildProjectNotificationEvent(args.envelope.event, context);+ case "prompt-version":+ return buildPromptChangeEvent(args.envelope, context);+ }+}++/** buildMonitorAlertEvent triggers the monitor's alert at the alert's severity, or resolves it on OK. */+function buildMonitorAlertEvent(+ alert: MonitorAlert,+ context: EventContext,+): PagerDutyEvent | undefined {+ if (alert.severity === "OK") return buildMonitorResolveEvent(alert, context);++ const severity = monitorSeverities[alert.severity];+ if (!severity) return undefined;+ return buildMonitorTriggerEvent(alert, severity, context);+}++/** buildMonitorResolveEvent resolves the monitor's open alert. */+function buildMonitorResolveEvent(+ alert: MonitorAlert,+ context: EventContext,+): PagerDutyEvent {+ return {+ kind: "alert",+ event: {+ routing_key: context.routingKey,+ event_action: "resolve",+ dedup_key: monitorDedupKey(context, alert.monitorId),+ },+ };+}++/** monitorDedupKey keys a monitor's alert so later triggers update its severity and OK resolves it. */+function monitorDedupKey(context: EventContext, monitorId: string): string {+ return `langfuse-${context.projectId}-${monitorId}`;+}++/** buildMonitorTriggerEvent triggers the monitor's alert for the alert's severity. */+function buildMonitorTriggerEvent(+ alert: MonitorAlert,+ severity: PagerDutySeverity,+ context: EventContext,+): PagerDutyEvent {+ return {+ kind: "alert",+ event: {+ routing_key: context.routingKey,+ event_action: "trigger",+ dedup_key: monitorDedupKey(context, alert.monitorId),+ ...clientFields(context.projectId),+ links: compactLinks([+ { href: alert.permalink, text: "View monitor in Langfuse" },+ { href: alert.dataPermalink, text: "View data in the alert window" },+ ]),+ payload: {+ summary: truncatePagerDutySummary(alert.message.title),+ source: source(context.projectName),+ severity,+ timestamp: alert.timestamp.toISOString(),+ class: "monitor-alert",+ component: alert.view,+ custom_details: {+ llmops: buildLlmOpsDetails(alert, context.projectName),+ langfuse: {+ severity: alert.severity,+ message: alert.message.body,+ filters: alert.filters,+ window: alert.window,+ },+ },+ },+ },+ };+}++/** clientFields names Langfuse as the sending client and links the project. */+function clientFields(+ projectId: string,+): Pick<PagerDutyAlertEvent, "client" | "client_url"> {+ return {+ client,+ client_url: safeUrl(() =>+ new URL(`project/${projectId}`, getProductBaseUrl()).toString(),+ ),+ };+}++/** safeUrl returns undefined when the Langfuse base URL is not configured. */+function safeUrl(build: () => string): string | undefined {+ try {+ return build();+ } catch {+ return undefined;+ }+}++/** compactLinks drops links whose URL is unknown. */+function compactLinks(+ links: { href: string | undefined; text: string }[],+): PagerDutyLink[] {+ return links.flatMap((link) => {+ if (!link.href) return [];+ return [{ href: link.href, text: link.text }];+ });+}++/** source identifies the Langfuse project as the affected system. */+function source(projectName: string): string {+ return `langfuse/${projectName}`;+}++/** buildLlmOpsDetails fills PagerDuty's LLMOps fields from the alert. */+function buildLlmOpsDetails(+ alert: MonitorAlert,+ projectName: string,+): Record<string, unknown> {+ const metricDescription = describeMetric(alert.metric);+ return {+ model_name: filteredModelName(alert.filters),+ project_name: projectName,+ monitor_name: alert.monitorName,+ monitor_id: alert.monitorId,+ monitor_alert_start: alert.fromTimestamp.toISOString(),+ monitor_alert_end: alert.toTimestamp.toISOString(),+ metric_description: metricDescription,+ metric_definition: metricDefinition(metricDescription, alert),+ metric_value: alert.metricValue ?? undefined,+ threshold_value: breachedThreshold(alert),+ };+}++/** describeMetric renders a metric as `aggregation(measure)`. */+function describeMetric(metric: MonitorAlert["metric"]): string | undefined {+ if (!metric) return undefined;+ return `${metric.aggregation}(${metric.measure})`;+}++/** filteredModelName returns the model a monitor filters on, when it filters on exactly one. */+function filteredModelName(+ filters: MonitorAlert["filters"],+): string | undefined {+ for (const filter of filters) {+ if (!modelFilterColumns.has(filter.column)) continue;+ if (filter.type === "string" && filter.operator === "=")+ return filter.value;+ if (+ filter.type === "stringOptions" &&+ filter.operator === "any of" &&+ filter.value.length === 1+ ) {+ return filter.value[0];+ }+ }+ return undefined;+}++/** metricDefinition states the metric, its view and the threshold rule. */+function metricDefinition(+ metricDescription: string | undefined,+ alert: MonitorAlert,+): string | undefined {+ if (!metricDescription) return undefined;+ const definition = `${metricDescription} over ${alert.view} per ${alert.window}`;+ if (!alert.thresholdOperator) return definition;+ return `${definition} ${alert.thresholdOperator} ${alert.alertThreshold}`;+}++/** breachedThreshold is the warning threshold for a warning and the alert threshold otherwise. */+function breachedThreshold(alert: MonitorAlert): number | undefined {+ if (alert.severity === "WARNING") return alert.warningThreshold ?? undefined;+ return alert.alertThreshold;+}++/** buildProjectNotificationEvent always triggers, keyed by event type and resource; these are never resolved automatically. */+function buildProjectNotificationEvent(+ event: ProjectNotificationEvent,+ context: EventContext,+): PagerDutyEvent {+ return {+ kind: "alert",+ event: {+ routing_key: context.routingKey,+ event_action: "trigger",+ dedup_key: `langfuse-${context.projectId}-${event.eventType}-${event.resourceId}`,+ ...clientFields(context.projectId),+ links: compactLinks([{ href: event.url, text: "Open in Langfuse" }]),+ payload: {+ summary: truncatePagerDutySummary(+ `${event.message} (${event.resourceName})`,+ ),+ source: source(context.projectName),+ severity: notificationSeverities[event.severity],+ timestamp: new Date().toISOString(),+ class: event.eventType,+ component: event.resourceName,+ custom_details: { langfuse: event },+ },+ },+ };+}++/** buildPromptChangeEvent records a prompt version change as a PagerDuty change event, which never pages anyone. */+function buildPromptChangeEvent(+ envelope: Extract<PagerDutyEnvelope, { type: "prompt-version" }>,+ context: EventContext,+): PagerDutyEvent {+ const { prompt } = envelope;+ return {+ kind: "change",+ event: {+ routing_key: context.routingKey,+ links: compactLinks([+ {+ href: safeUrl(() =>+ buildPromptUrl({+ projectId: context.projectId,+ name: prompt.name,+ version: prompt.version,+ }),+ ),+ text: "View prompt in Langfuse",+ },+ ]),+ payload: {+ summary: truncatePagerDutySummary(+ `Langfuse prompt "${prompt.name}" version ${prompt.version} ${envelope.action}`,+ ),+ source: source(context.projectName),+ timestamp: new Date().toISOString(),+ custom_details: {+ prompt_name: prompt.name,+ prompt_version: prompt.version,+ action: envelope.action,+ labels: prompt.labels,+ tags: prompt.tags,+ commit_message: prompt.commitMessage,+ },+ },+ },+ };+}++/** PagerDutyEvent is an alert event for `/v2/enqueue` or a change event for `/v2/change/enqueue`. */+export type PagerDutyEvent =+ | { kind: "alert"; event: PagerDutyAlertEvent }+ | { kind: "change"; event: PagerDutyChangeEvent };++/** PagerDutyEnvelope is the validated webhook envelope the worker delivers. */+type PagerDutyEnvelope =+ | {+ type: "monitor-alert";+ payload: MonitorAlert;+ }+ | {+ type: "project-notification";+ event: ProjectNotificationEvent;+ }+ | {+ type: "prompt-version";+ action: string;+ prompt: {+ name: string;+ version: number;+ labels: string[];+ tags: string[];+ commitMessage: string | null;+ };+ };++/** ProjectNotificationEvent is the subset of a project notification PagerDuty needs. */+type ProjectNotificationEvent = {+ eventType: string;+ severity: "ALERT" | "WARNING" | "INFO";+ resourceId: string;+ resourceName: string;+ message: string;+ url?: string;+};++/** EventContext carries what every event needs besides its payload. */+type EventContext = {+ projectId: string;+ projectName: string;+ routingKey: string;+};diff --git a/worker/src/queues/webhooks.ts b/worker/src/queues/webhooks.tsindex 93313fe68c..e5f318182b 100644--- a/worker/src/queues/webhooks.ts+++ b/worker/src/queues/webhooks.ts@@ -9,8 +9,11 @@ import {isSlackActionConfig,isWebhookAction,isGitHubDispatchAction,+ isPagerDutyActionConfig,+ isBaseError,type AutomationDomain,type ActionDomainWithSecrets,+ type PagerDutyActionConfigWithSecrets,} from "@langfuse/shared";import { decrypt, createSignatureHeader } from "@langfuse/shared/encryption";import { Prisma, prisma } from "@langfuse/shared/src/db";@@ -23,6 +26,7 @@ import {type WebhookValidationWhitelist,WEBHOOK_URL_VALIDATION_LOG_CONTEXT,buildWebhookRequestHeaders,+ recordIncrement,} from "@langfuse/shared/src/server";import {TQueueJobTypes,@@ -38,14 +42,23 @@ import {ProjectNotificationWebhookQueueEventSchema,buildProjectNotificationSlackMessage,} from "@langfuse/shared/src/server";+import { type MonitorAlert } from "@langfuse/shared/monitors";import {MonitorWebhookQueueEventSchema,buildMonitorAlertSlackMessage,} from "@langfuse/shared/monitors/server";+import {+ pagerDutyClient,+ type PagerDutyRegion,+} from "@langfuse/shared/pagerduty/server";import { Processor, Job } from "bullmq";import { backOff } from "exponential-backoff";import { env } from "../env";import { SlackMessageBuilder } from "../features/slack/slackMessageBuilder";+import {+ buildPagerDutyEvent,+ type PagerDutyEvent,+} from "../features/pagerduty/buildPagerDutyEvent";/** WEBHOOK_ADDRESS_ATTEMPT_TIMEOUT_MS lets a retransmitted SYN finish before Node closes the socket. */const WEBHOOK_ADDRESS_ATTEMPT_TIMEOUT_MS = 2_000;@@ -90,6 +103,9 @@ async function resetAutomationFailures(args: {await redis.del(automationFailureKey(args.projectId, args.automationId));}+/** WebhookOutboundPayload is the validated HTTP body of a WebhookInput. */+type WebhookOutboundPayload = ReturnType<typeof buildWebhookOutboundPayload>;+/** buildWebhookOutboundPayload validates and returns the HTTP body for a WebhookInput, discriminating on payload.type. monitor-alert: the processor already built the unified envelope — pass through. prompt-version: wrap at dispatch time with id = executionId and timestamp = now. */function buildWebhookOutboundPayload(input: WebhookInput) {if (input.payload.type === "monitor-alert") {@@ -166,6 +182,17 @@ export const executeWebhook = async (return;}+ if (+ await isStaleMonitorAlert({+ projectId,+ automationId,+ actionType: automation.action.type,+ input,+ })+ ) {+ return;+ }+// Route to appropriate handler based on action typeif (automation.action.type === "WEBHOOK") {await executeWebhookAction({@@ -184,6 +211,8 @@ export const executeWebhook = async (automation,skipValidation: options?.skipValidation,});+ } else if (automation.action.type === "PAGERDUTY") {+ await executePagerDutyAction({ input, automation });} else {throw new InternalServerError(`Unsupported action type: ${automation.action.type}`,@@ -199,6 +228,90 @@ export const executeWebhook = async (}};+/** isStaleMonitorAlert prevents older queued alerts from overwriting newer published alerts by reporting which jobs to drop. */+async function isStaleMonitorAlert({+ projectId,+ automationId,+ actionType,+ input,+}: {+ projectId: string;+ automationId: string;+ actionType: string;+ input: WebhookInput;+}): Promise<boolean> {+ if (input.payload.type !== "monitor-alert") return false;+ const envelope = buildWebhookOutboundPayload(input);+ if (envelope.type !== "monitor-alert") return false;+ const alert = envelope.payload;++ const reason = await getStaleMonitorAlertReason({ projectId, alert });+ if (!reason) return false;++ recordStaleMonitorAlertDrop({+ projectId,+ automationId,+ actionType,+ monitorId: alert.monitorId,+ reason,+ });+ return true;+}++/** getStaleMonitorAlertReason says why an alert is stale, or undefined when it is current. */+async function getStaleMonitorAlertReason({+ projectId,+ alert,+}: {+ projectId: string;+ alert: MonitorAlert;+}): Promise<StaleMonitorAlertReason | undefined> {+ const monitor = await prisma.monitor.findUnique({+ where: { id: alert.monitorId },+ select: { projectId: true, alertedAt: true },+ });+ if (!monitor || monitor.projectId !== projectId) return "monitor_deleted";+ // The processor publishes before it saves alertedAt, so only a later alertedAt is newer.+ if (+ alert.alertedAt &&+ monitor.alertedAt &&+ monitor.alertedAt > alert.alertedAt+ ) {+ return "newer_alert";+ }+ return undefined;+}++/** recordStaleMonitorAlertDrop logs a dropped stale alert and counts it by reason and action type. */+function recordStaleMonitorAlertDrop({+ projectId,+ automationId,+ actionType,+ monitorId,+ reason,+}: {+ projectId: string;+ automationId: string;+ actionType: string;+ monitorId: string;+ reason: StaleMonitorAlertReason;+}): void {+ logger.info("Dropping stale monitor alert", {+ projectId,+ automationId,+ monitorId,+ reason,+ });+ recordIncrement(+ "langfuse.queue.webhook_queue.stale_monitor_alerts_dropped",+ 1,+ {+ reason,+ actionType,+ },+ );+}+/*** Shared HTTP execution logic for webhook and GitHub dispatch actions* Handles: fetch with timeout, retry logic, status validation, execution tracking@@ -521,12 +634,12 @@ async function executeWebhookAction({const webhookConfig = actionConfig.config;- const validated = buildWebhookOutboundPayload(input);+ const envelope = buildWebhookOutboundPayload(input);let webhookPayload: string;if (input.payload.type === "prompt-version") {// Prompt-version contract: prompt key always last.- const { prompt, user, ...otherFields } = validated as Extract<- typeof validated,+ const { prompt, user, ...otherFields } = envelope as Extract<+ typeof envelope,{ type: "prompt-version" }>;webhookPayload = JSON.stringify({@@ -536,7 +649,7 @@ async function executeWebhookAction({});} else {// monitor-alert: post the envelope verbatim.- webhookPayload = JSON.stringify(validated);+ webhookPayload = JSON.stringify(envelope);}let requestHeaders: Record<string, string>;@@ -609,12 +722,12 @@ async function executeGitHubDispatchAction({const githubConfig = actionConfig.config;const eventType = githubConfig.eventType;- const validated = buildWebhookOutboundPayload(input);+ const envelope = buildWebhookOutboundPayload(input);let githubPayload: string;if (input.payload.type === "prompt-version") {- const { prompt, user, ...otherFields } = validated as Extract<- typeof validated,+ const { prompt, user, ...otherFields } = envelope as Extract<+ typeof envelope,{ type: "prompt-version" }>;const fullGithubPayload = JSON.stringify({@@ -647,8 +760,8 @@ async function executeGitHubDispatchAction({});} else {// monitor-alert: post the envelope as client_payload.- const monitorEnvelope = validated as Extract<- typeof validated,+ const monitorEnvelope = envelope as Extract<+ typeof envelope,{ type: "monitor-alert" }>;const fullGithubPayload = JSON.stringify({@@ -761,7 +874,11 @@ async function executeSlackAction({});} catch (error) {logger.error("Error executing Slack action", error);- await slackMonitorAlertFailure({ projectId, automation });+ await monitorAlertDeliveryFailure({+ projectId,+ automation,+ channel: "slack",+ });}return;}@@ -929,6 +1046,157 @@ async function executeSlackAction({}}+/**+ * Execute PagerDuty action. Events go only to the fixed PagerDuty hosts of the+ * routing key's region. 429, 5xx and network errors are retried in-process+ * like HTTP actions; a failure that remains counts toward trigger auto-disable.+ */+async function executePagerDutyAction({+ input,+ automation,+}: {+ input: WebhookInput;+ automation: NonNullable<Awaited<ReturnType<typeof getAutomationById>>>;+}) {+ const { projectId, executionId } = input;+ const executionStart = new Date();++ const actionConfig = await getActionByIdWithSecrets({+ projectId,+ actionId: automation.action.id,+ });+ if (!actionConfig || !isPagerDutyActionConfig(actionConfig.config)) {+ throw new InternalServerError("PagerDuty action config not found");+ }+ const config = actionConfig.config;++ try {+ const envelope = buildWebhookOutboundPayload(input);+ const event = await buildPagerDutyDelivery({ input, envelope, config });+ if (!event) {+ logger.info(+ `No PagerDuty event for ${input.payload.type} of automation ${automation.id} in project ${projectId}`,+ );+ return;+ }+ await sendPagerDutyEvent(event, config.region);+ } catch (error) {+ logger.error("Error executing PagerDuty action", error);+ await recordPagerDutyFailure({ input, automation, executionStart, error });+ return;+ }++ if (input.payload.type === "monitor-alert") {+ await resetAutomationFailures({ projectId, automationId: automation.id });+ return;+ }+ await prisma.automationExecution.update({+ where: {+ projectId,+ triggerId: automation.trigger.id,+ actionId: automation.action.id,+ id: executionId,+ },+ data: {+ status: ActionExecutionStatus.COMPLETED,+ startedAt: executionStart,+ finishedAt: new Date(),+ },+ });+}++/** buildPagerDutyDelivery decrypts the routing key and maps the job payload to its PagerDuty event. */+async function buildPagerDutyDelivery({+ input,+ envelope,+ config,+}: {+ input: WebhookInput;+ envelope: WebhookOutboundPayload;+ config: PagerDutyActionConfigWithSecrets;+}): Promise<PagerDutyEvent | undefined> {+ const project = await prisma.project.findUniqueOrThrow({+ where: { id: input.projectId },+ select: { name: true },+ });+ return buildPagerDutyEvent({+ envelope,+ projectId: input.projectId,+ projectName: project.name,+ routingKey: decrypt(config.routingKey),+ });+}++/** sendPagerDutyEvent posts the event to the alert or change endpoint of the key's region. */+async function sendPagerDutyEvent(+ event: PagerDutyEvent,+ region: PagerDutyRegion,+) {+ if (event.kind === "change") {+ await pagerDutyClient.change({ region, event: event.event });+ return;+ }+ await pagerDutyClient.alert({ region, event: event.event });+}++/** recordPagerDutyFailure records a non-retryable delivery failure and auto-disables the trigger like the other action types. */+async function recordPagerDutyFailure({+ input,+ automation,+ executionStart,+ error,+}: {+ input: WebhookInput;+ automation: NonNullable<Awaited<ReturnType<typeof getAutomationById>>>;+ executionStart: Date;+ error: unknown;+}) {+ const { projectId, executionId } = input;+ if (input.payload.type === "monitor-alert") {+ await monitorAlertDeliveryFailure({+ projectId,+ automation,+ channel: "pagerduty",+ });+ return;+ }++ await prisma.$transaction(async (tx) => {+ await tx.automationExecution.update({+ where: {+ id: executionId,+ projectId,+ triggerId: automation.trigger.id,+ actionId: automation.action.id,+ },+ data: {+ status: ActionExecutionStatus.ERROR,+ startedAt: executionStart,+ finishedAt: new Date(),+ error: error instanceof Error ? error.message : "Unknown error",+ output: isBaseError(error) ? { httpStatus: error.httpCode } : undefined,+ },+ });++ // Project notifications record the error but never auto-disable.+ if (input.payload.type === "project-notification") return;++ const consecutiveFailures = await getConsecutiveAutomationFailures({+ automationId: automation.id,+ projectId,+ });+ if (consecutiveFailures >= automationFailureThreshold - 1) {+ await tx.trigger.update({+ where: { id: automation.trigger.id, projectId },+ data: { status: JobConfigState.INACTIVE },+ });+ logger.warn(+ `Automation ${automation.trigger.id} disabled after ${consecutiveFailures} consecutive failures in project ${projectId} (pagerduty)`,+ );+ }+ });+}+/** sendSlackMonitorAlert builds and posts a monitor-alert to Slack, then clears the failure streak. */async function sendSlackMonitorAlert({payload,@@ -984,13 +1252,15 @@ async function sendSlackProjectNotification({});}-/** slackMonitorAlertFailure tracks a failed monitor-alert delivery and auto-disables the trigger past the failure threshold. */-async function slackMonitorAlertFailure({+/** monitorAlertDeliveryFailure tracks a failed monitor-alert delivery and auto-disables the trigger past the failure threshold. */+async function monitorAlertDeliveryFailure({projectId,automation,+ channel,}: {projectId: string;automation: NonNullable<Awaited<ReturnType<typeof getAutomationById>>>;+ channel: "slack" | "pagerduty";}) {const count = await incrementAutomationFailure({projectId,@@ -1006,7 +1276,7 @@ async function slackMonitorAlertFailure({automationId: automation.id,});logger.warn(- `Automation ${automation.trigger.id} disabled after ${count} consecutive failures in project ${projectId} (monitor-alert/slack)`,+ `Automation ${automation.trigger.id} disabled after ${count} consecutive failures in project ${projectId} (monitor-alert/${channel})`,);}}@@ -1038,3 +1308,6 @@ const setActionLastFailingExecutionId = async ({AND project_id = ${projectId}`;};++/** StaleMonitorAlertReason is why a queued monitor alert was dropped. */+type StaleMonitorAlertReason = "monitor_deleted" | "newer_alert";