🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import {
describe,
it,
expect,
beforeEach,
beforeAll,
afterAll,
afterEach,
vi,
} from "vitest";
import { v4 } from "uuid";
import dns from "node:dns/promises";
import { backOff } from "exponential-backoff";
import {
ActionExecutionStatus,
JobConfigState,
PromptDomainSchema,
WebhookActionConfigWithSecrets,
} from "@langfuse/shared";
import {
WebhookInput,
createOrgProjectAndApiKey,
getActionById,
getActionByIdWithSecrets,
logger,
redis,
} from "@langfuse/shared/src/server";
import { prisma } from "@langfuse/shared/src/db";
import {
decrypt,
encrypt,
generateWebhookSignature,
} from "@langfuse/shared/encryption";
import { generateWebhookSecret } from "@langfuse/shared/encryption";
import { setupServer } from "msw/node";
import { http, HttpResponse } from "msw";
import { executeWebhook } from "../queues/webhooks";
const outboundWhitelistBuilders = vi.hoisted(() => ({
webhookWhitelistFromEnv: vi.fn(),
whitelistFromEnv: vi.fn(),
realWebhookWhitelistFromEnv: (): {
hosts: string[];
ips: string[];
ip_ranges: string[];
allowedHosts?: string[];
} => ({ hosts: [], ips: [], ip_ranges: [] }),
realWhitelistFromEnv: (): {
hosts: string[];
ips: string[];
ip_ranges: string[];
allowedHosts?: string[];
} => ({ hosts: [], ips: [], ip_ranges: [] }),
}));
vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
const actual =
await importOriginal<typeof import("@langfuse/shared/src/server")>();
outboundWhitelistBuilders.realWebhookWhitelistFromEnv =
actual.webhookWhitelistFromEnv;
outboundWhitelistBuilders.realWhitelistFromEnv = actual.whitelistFromEnv;
outboundWhitelistBuilders.webhookWhitelistFromEnv.mockImplementation(
actual.webhookWhitelistFromEnv,
);
outboundWhitelistBuilders.whitelistFromEnv.mockImplementation(
actual.whitelistFromEnv,
);
return {
...actual,
webhookWhitelistFromEnv: outboundWhitelistBuilders.webhookWhitelistFromEnv,
whitelistFromEnv: outboundWhitelistBuilders.whitelistFromEnv,
};
});
vi.mock("exponential-backoff", async (importOriginal) => {
const actual = await importOriginal<typeof import("exponential-backoff")>();
return {
...actual,
// State-transition tests exercise every HTTP attempt without waiting between them.
// The endpoint-error test restores the real delays to cover the production policy.
backOff: vi.fn((...args: Parameters<typeof actual.backOff>) =>
actual.backOff(args[0], { ...args[1], startingDelay: 0 }),
),
};
});
// Mock webhook server for testing HTTP requests
class WebhookTestServer {
private server;
private receivedRequests: Array<{
url: string;
method: string;
headers: Record<string, string>;
body: string;
}> = [];
pagerDutyStatus = 202;
constructor() {
const recordPagerDutyEvent = async ({ request }: { request: Request }) => {
this.receivedRequests.push({
url: request.url,
method: request.method,
headers: Object.fromEntries(request.headers.entries()),
body: JSON.stringify(await request.json()),
});
return HttpResponse.json(
{ status: "success", message: "Event processed" },
{ status: this.pagerDutyStatus },
);
};
this.server = setupServer(
http.post("https://events.pagerduty.com/*", recordPagerDutyEvent),
http.post("https://events.eu.pagerduty.com/*", recordPagerDutyEvent),
// Default success response
http.post("https://webhook.example.com/*", async ({ request }) => {
this.receivedRequests.push({
url: request.url,
method: request.method,
headers: Object.fromEntries(request.headers.entries()),
body: JSON.stringify(await request.json()),
});
return HttpResponse.json({ success: true }, { status: 200 });
}),
// Error response endpoint
http.post("https://webhook-error.example.com/*", async ({ request }) => {
this.receivedRequests.push({
url: request.url,
method: request.method,
headers: Object.fromEntries(request.headers.entries()),
body: JSON.stringify(await request.json()),
});
return HttpResponse.json(
{ error: "Internal Server Error" },
{ status: 500 },
);
}),
// 201 Created response endpoint (GitLab use case)
http.post("https://webhook-201.example.com/*", async ({ request }) => {
this.receivedRequests.push({
url: request.url,
method: request.method,
headers: Object.fromEntries(request.headers.entries()),
body: JSON.stringify(await request.json()),
});
return HttpResponse.json({ success: true }, { status: 201 });
}),
// Timeout endpoint
http.post("https://webhook-timeout.example.com/*", () => {
return HttpResponse.error();
}),
);
}
setup() {
this.server.listen();
}
reset() {
this.receivedRequests = [];
this.pagerDutyStatus = 202;
this.server.resetHandlers();
}
teardown() {
this.server.close();
}
getReceivedRequests() {
return this.receivedRequests;
}
getLastRequest() {
return this.receivedRequests[this.receivedRequests.length - 1];
}
}
const webhookServer = new WebhookTestServer();
/** createMonitor writes a minimal Monitor row so monitor-alert jobs pass the stale-alert guard. */
async function createMonitor(projectId: string): Promise<string> {
const monitor = await prisma.monitor.create({
data: {
projectId,
name: "Test monitor",
view: "OBSERVATIONS",
filters: [],
metric: { measure: "count", aggregation: "count" },
windowMs: 5n * 60_000n,
cadenceMs: 60_000n,
thresholdOperator: "GT",
alertThreshold: 100,
noData: { mode: "SHOW_NO_DATA" },
renotify: { mode: "OFF" },
schedulerBatchId: 0n,
severity: "ALERT",
},
});
return monitor.id;
}
const GITHUB_REPOSITORY_DISPATCH_TRUNCATION_MARKER =
"[TRUNCATED: GitHub repository_dispatch payload exceeded size limit]";
const GITHUB_REPOSITORY_DISPATCH_MAX_PAYLOAD_BYTES = 64 * 1024;
describe("Webhook Integration Tests", () => {
let projectId: string;
let triggerId: string;
let actionId: string;
let automationId: string;
let promptId: string;
let executionId: string;
let monitorId: string;
beforeAll(() => {
webhookServer.setup();
});
beforeEach(async () => {
webhookServer.reset();
// Create test project
({ projectId } = await createOrgProjectAndApiKey());
// Create test prompt
promptId = v4();
await prisma.prompt.create({
data: {
id: promptId,
projectId,
name: "test-prompt",
version: 1,
type: "text",
prompt: { text: "Hello {{name}}" },
createdBy: "test-user",
},
});
// Create test trigger
triggerId = v4();
await prisma.trigger.create({
data: {
id: triggerId,
projectId,
eventSource: "prompt-version",
eventActions: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
},
});
// Create test action
actionId = v4();
const { secretKey, displaySecretKey } = generateWebhookSecret();
await prisma.action.create({
data: {
id: actionId,
projectId,
type: "WEBHOOK",
config: {
type: "WEBHOOK",
url: "https://webhook.example.com/test",
headers: {
"X-Custom-Header-2": "test-value-2",
"X-Custom-Header": "test-value",
},
apiVersion: { prompt: "v1" },
secretKey: encrypt(secretKey),
displaySecretKey,
},
},
});
// Link trigger to action
automationId = v4();
await prisma.automation.create({
data: {
id: automationId,
projectId,
triggerId,
actionId,
name: "Test Automation",
},
});
executionId = v4();
monitorId = await createMonitor(projectId);
});
afterEach(() => {
webhookServer.reset();
});
afterAll(() => {
webhookServer.teardown();
});
describe("skill version delivery", () => {
it.each([
{ github: false, oversized: false },
{ github: true, oversized: true },
])(
"serializes skill metadata (github=$github, oversized=$oversized)",
async ({ github, oversized }) => {
if (github) {
await prisma.action.update({
where: { id: actionId, projectId },
data: {
type: "GITHUB_DISPATCH",
config: {
type: "GITHUB_DISPATCH",
url: "https://webhook.example.com/dispatches",
eventType: "skill-update",
githubToken: encrypt("github-test-token"),
displayGitHubToken: "github-...",
},
},
});
}
const skillId = v4();
const webhookInput: WebhookInput = {
projectId,
automationId,
executionId,
payload: {
type: "skill-version",
action: "created",
skill: {
id: skillId,
projectId,
name: "test-skill",
description: "Test skill",
version: 2,
frontmatter: oversized ? { detail: "x".repeat(70_000) } : {},
tags: [],
labels: ["latest"],
commitMessage: null,
createdBy: "test-user",
createdAt: new Date("2026-01-01T00:00:00Z"),
updatedAt: new Date("2026-01-01T00:00:00Z"),
files: [
{
id: "file-id",
blobId: "blob-id",
path: "SKILL.md",
sha256Hash: "file-hash",
contentType: "text/markdown",
contentLength: 42,
},
],
},
user: {
id: "internal-user",
name: "Test user",
email: "[email protected]",
},
},
};
Object.assign(webhookInput.payload.skill.files[0], {
content: "must not be exported",
});
await prisma.automationExecution.create({
data: {
id: executionId,
projectId,
triggerId,
actionId,
automationId,
status: ActionExecutionStatus.PENDING,
sourceId: skillId,
input: {},
},
});
await executeWebhook(webhookInput, { skipValidation: true });
const requests = webhookServer.getReceivedRequests();
expect(requests).toHaveLength(1);
const request = requests[0];
const parsed = JSON.parse(request.body);
const payload = github ? parsed.client_payload : parsed;
expect(payload).toMatchObject({
id: executionId,
type: "skill-version",
action: "created",
apiVersion: "v1",
skill: { id: skillId, projectId, name: "test-skill", version: 2 },
});
expect(payload).not.toHaveProperty("prompt");
expect(payload.user?.id).toBeUndefined();
if (github) {
expect(parsed.event_type).toBe("skill-update");
expect(Buffer.byteLength(request.body, "utf8")).toBeLessThan(
64 * 1024,
);
}
if (oversized) {
expect(payload.truncation.payloadTruncated).toBe(true);
expect(payload.skill.files).toBeUndefined();
} else {
expect(payload.skill.createdAt).toBe("2026-01-01T00:00:00.000Z");
expect(payload.skill.files).toMatchObject([
{ path: "SKILL.md", sha256Hash: "file-hash", contentLength: 42 },
]);
expect(payload.skill.files[0].content).toBeUndefined();
}
expect(
await prisma.automationExecution.findUnique({
where: { id: executionId },
}),
).toMatchObject({ status: ActionExecutionStatus.COMPLETED });
},
);
});
describe("executeWebhook function", () => {
it("should execute webhook successfully with signature", async () => {
// Get the full prompt for the payload
const fullPrompt = await prisma.prompt.findUnique({
where: { id: promptId },
});
const webhookInput: WebhookInput = {
projectId,
automationId,
executionId,
payload: {
prompt: PromptDomainSchema.parse(fullPrompt),
action: "created",
type: "prompt-version",
user: {
id: "user-123",
name: "Test User",
email: "[email protected]",
},
},
};
await prisma.automationExecution.create({
data: {
id: executionId,
projectId,
triggerId,
automationId,
actionId,
status: ActionExecutionStatus.PENDING,
sourceId: webhookInput.executionId,
input: webhookInput,
},
});
await executeWebhook(webhookInput, { skipValidation: true });
// Verify webhook request was made
const requests = webhookServer.getReceivedRequests();
expect(requests).toHaveLength(1);
const request = requests[0];
expect(request.url).toBe("https://webhook.example.com/test");
expect(request.method).toBe("POST");
expect(request.headers["x-custom-header-2"]).toBe("test-value-2");
expect(request.headers["x-custom-header"]).toBe("test-value");
expect(request.headers["x-langfuse-signature"]).toMatch(
/^t=\d+,v1=[a-f0-9]+$/,
);
// check signature
const signature = request.headers["x-langfuse-signature"];
const payload = JSON.parse(request.body);
const action = await prisma.action.findUnique({
where: { id: actionId },
});
const secretKey = (action?.config as WebhookActionConfigWithSecrets)
?.secretKey;
if (!secretKey) {
throw new Error("Action has no secret key");
}
const decryptedSecret = decrypt(secretKey);
// Extract timestamp from the actual signature to avoid timing issues
const timestampMatch = signature.match(/^t=(\d+),v1=/);
if (!timestampMatch) {
throw new Error("Invalid signature format");
}
const timestamp = parseInt(timestampMatch[1]);
// Generate expected signature using the same timestamp
const expectedSignatureHash = generateWebhookSignature(
JSON.stringify(payload),
timestamp,
decryptedSecret,
);
const expectedSignature = `t=${timestamp},v1=${expectedSignatureHash}`;
expect(signature).toBe(expectedSignature);
expect(payload.id).toBe(webhookInput.executionId);
expect(payload.type).toBe("prompt-version");
expect(payload.action).toBe("created");
expect(payload.prompt.name).toBe("test-prompt");
expect(payload.prompt.version).toBe(1);
expect(payload.prompt.createdBy).toBe("test-user");
expect(payload.timestamp).toBeDefined();
expect(payload.prompt.createdAt).toBeDefined();
expect(payload.prompt.updatedAt).toBeDefined();
// Verify prompt is the last field in the payload
const payloadKeys = Object.keys(payload);
expect(payloadKeys[payloadKeys.length - 1]).toBe("prompt");
// Verify database execution record was updated
const execution = await prisma.automationExecution.findUnique({
where: { id: executionId },
});
expect(execution?.status).toBe(ActionExecutionStatus.COMPLETED);
expect(execution?.output).toBeNull();
expect(execution?.startedAt).toBeDefined();
expect(execution?.finishedAt).toBeDefined();
});
it("should fail webhook execution if secret key does not exist and retry the bull job", async () => {
const fullPrompt = await prisma.prompt.findUnique({
where: { id: promptId },
});
const executionId = v4();
await prisma.action.update({
where: { id: actionId },
data: {
config: {
type: "WEBHOOK",
url: "https://webhook-error.example.com/test",
headers: { "Content-Type": "application/json" },
apiVersion: { prompt: "v1" },
secretKey: null, // Explicitly set secret to null
},
},
});
await prisma.automationExecution.create({
data: {
id: executionId,
projectId,
triggerId,
automationId,
actionId,
status: ActionExecutionStatus.PENDING,
sourceId: v4(),
input: {
promptName: "test-prompt",
promptVersion: 1,
action: "created",
type: "prompt-version",
},
},
});
const webhookInput: WebhookInput = {
projectId,
automationId,
executionId,
payload: {
prompt: PromptDomainSchema.parse(fullPrompt),
action: "created",
type: "prompt-version",
user: {
id: "user-123",
name: "Test User",
email: "[email protected]",
},
},
};
await expect(
executeWebhook(webhookInput, { skipValidation: true }),
).rejects.toThrow("Action config is not a valid webhook configuration");
const execution = await prisma.automationExecution.findUnique({
where: { id: executionId },
});
expect(execution?.status).toBe(ActionExecutionStatus.PENDING);
});
it("should accept 201 status code as success (GitLab use case)", async () => {
// Get the full prompt for the payload
const fullPrompt = await prisma.prompt.findUnique({
where: { id: promptId },
});
const action = await prisma.action.findUnique({
where: { id: actionId },
});
if (!action) {
throw new Error("Action not found");
}
// Update action to use 201 endpoint
await prisma.action.update({
where: { id: actionId },
data: {
projectId,
type: "WEBHOOK",
config: {
...(action.config as WebhookActionConfigWithSecrets),
url: "https://webhook-201.example.com/test",
},
},
});
await prisma.automationExecution.create({
data: {
id: executionId,
projectId,
triggerId,
automationId,
actionId,
status: ActionExecutionStatus.PENDING,
sourceId: v4(),
input: {
promptName: "test-prompt",
promptVersion: 1,
action: "created",
type: "prompt-version",
},
},
});
const webhookInput: WebhookInput = {
projectId,
automationId,
executionId,
payload: {
prompt: PromptDomainSchema.parse(fullPrompt),
action: "created",
type: "prompt-version",
user: {
id: "user-123",
name: "Test User",
email: "[email protected]",
},
},
};
await executeWebhook(webhookInput, { skipValidation: true });
// Verify webhook request was made
const requests = webhookServer.getReceivedRequests();
expect(requests).toHaveLength(1);
const request = requests[0];
expect(request.url).toBe("https://webhook-201.example.com/test");
expect(request.method).toBe("POST");
// Verify execution was marked as completed (not error)
const execution = await prisma.automationExecution.findUnique({
where: { id: executionId },
});
expect(execution?.status).toBe(ActionExecutionStatus.COMPLETED);
expect(execution?.startedAt).toBeDefined();
expect(execution?.finishedAt).toBeDefined();
});
it("should handle webhook endpoint returning error", async () => {
const actual = await vi.importActual<
typeof import("exponential-backoff")
>("exponential-backoff");
vi.mocked(backOff).mockImplementationOnce(actual.backOff);
vi.mocked(backOff).mockClear();
const fullPrompt = await prisma.prompt.findUnique({
where: { id: promptId },
});
const action = await prisma.action.findUnique({
where: { id: actionId },
});
if (!action) {
throw new Error("Action not found");
}
await prisma.action.update({
where: { id: actionId },
data: {
projectId,
type: "WEBHOOK",
config: {
...(action.config as WebhookActionConfigWithSecrets),
url: "https://webhook-error.example.com/test",
},
},
});
await prisma.automationExecution.create({
data: {
id: executionId,
projectId,
triggerId,
automationId,
actionId,
status: ActionExecutionStatus.PENDING,
sourceId: v4(),
input: {
promptName: "test-prompt",
promptVersion: 1,
action: "created",
type: "prompt-version",
},
},
});
// Create action pointing to error endpoint
const webhookInput: WebhookInput = {
projectId,
automationId,
executionId,
payload: {
prompt: PromptDomainSchema.parse(fullPrompt),
action: "created",
type: "prompt-version",
user: {
id: "user-123",
name: "Test User",
email: "[email protected]",
},
},
};
await executeWebhook(webhookInput, { skipValidation: true });
expect(backOff).toHaveBeenCalledWith(expect.any(Function), {
numOfAttempts: 4,
});
expect(webhookServer.getReceivedRequests()).toHaveLength(4);
// Verify execution was marked as error
const execution = await prisma.automationExecution.findUnique({
where: { id: executionId },
});
expect(execution?.status).toBe(ActionExecutionStatus.ERROR);
expect(execution?.error).toContain(
`Webhook does not return 2xx status: failed with status 500 for url https://webhook-error.example.com/test and project ${projectId}`,
);
expect(execution?.output).toMatchObject({
httpStatus: 500,
responseBody: '{"error":"Internal Server Error"}',
});
});
it("sends a query-string secret and keeps it out of logs and the execution error", async () => {
const signedUrl =
"https://webhook-error.example.com/test?api-version=2016-06-01&sig=super-secret-sig";
const action = await prisma.action.findUnique({
where: { id: actionId },
});
if (!action) {
throw new Error("Action not found");
}
await prisma.action.update({
where: { id: actionId },
data: {
projectId,
type: "WEBHOOK",
config: {
...(action.config as WebhookActionConfigWithSecrets),
url: signedUrl,
},
},
});
await prisma.automationExecution.create({
data: {
id: executionId,
projectId,
triggerId,
automationId,
actionId,
status: ActionExecutionStatus.PENDING,
sourceId: v4(),
input: {
promptName: "test-prompt",
promptVersion: 1,
action: "created",
type: "prompt-version",
},
},
});
const info = vi.spyOn(logger, "info");
const warn = vi.spyOn(logger, "warn");
const error = vi.spyOn(logger, "error");
try {
await executeWebhook(
{
projectId,
automationId,
executionId,
payload: {
prompt: PromptDomainSchema.parse(
await prisma.prompt.findUnique({ where: { id: promptId } }),
),
action: "created",
type: "prompt-version",
user: {
id: "user-123",
name: "Test User",
email: "[email protected]",
},
},
},
{ skipValidation: true },
);
const requests = webhookServer.getReceivedRequests();
expect(requests.length).toBeGreaterThan(0);
expect(requests.every((request) => request.url === signedUrl)).toBe(
true,
);
const execution = await prisma.automationExecution.findUnique({
where: { id: executionId },
});
expect(execution?.error).toContain(
"https://webhook-error.example.com/test",
);
expect(execution?.error).not.toContain("super-secret-sig");
const logged = [info, warn, error]
.flatMap((spy) => spy.mock.calls)
.map((call) =>
call
.map((arg) => {
if (arg instanceof Error) {
return `${arg.name} ${arg.message} ${arg.stack ?? ""} ${JSON.stringify(arg)}`;
}
return typeof arg === "string" ? arg : JSON.stringify(arg);
})
.join(" "),
)
.join("\n");
expect(logged).toContain("https://webhook-error.example.com/test");
expect(logged).not.toContain("super-secret-sig");
} finally {
info.mockRestore();
warn.mockRestore();
error.mockRestore();
}
});
it("should disable trigger after 5 consecutive failures and store lastFailingExecutionId", async () => {
const fullPrompt = await prisma.prompt.findUnique({
where: { id: promptId },
});
const action = await prisma.action.findUnique({
where: { id: actionId },
});
if (!action) {
throw new Error("Action not found");
}
const secretHeaderValue = "secret-api-key-value";
const encryptedSecretHeaderValue = encrypt(secretHeaderValue);
await prisma.action.update({
where: { id: actionId },
data: {
projectId,
type: "WEBHOOK",
config: {
...(action.config as WebhookActionConfigWithSecrets),
url: "https://webhook-error.example.com/test",
requestHeaders: {
"x-secret-api-key": {
secret: true,
value: encryptedSecretHeaderValue,
},
},
displayHeaders: {
"x-secret-api-key": { secret: true, value: "secr...alue" },
},
},
},
});
// Execute webhook 5 times to trigger consecutive failures
for (let i = 0; i < 5; i++) {
const executionId = v4();
await prisma.automationExecution.create({
data: {
id: executionId,
projectId,
triggerId,
automationId,
actionId,
status: ActionExecutionStatus.PENDING,
sourceId: v4(),
input: {
promptName: "test-prompt",
promptVersion: 1,
action: "created",
type: "prompt-version",
},
},
});
const webhookInput: WebhookInput = {
projectId,
automationId,
executionId,
payload: {
prompt: PromptDomainSchema.parse(fullPrompt),
action: "created",
type: "prompt-version",
user: {
id: "user-123",
name: "Test User",
email: "[email protected]",
},
},
};
await executeWebhook(webhookInput, { skipValidation: true });
// Verify execution was marked as error
const execution = await prisma.automationExecution.findUnique({
where: { id: executionId },
});
expect(execution?.status).toBe(ActionExecutionStatus.ERROR);
expect(execution?.error).toContain(
`Webhook does not return 2xx status: failed with status 500 for url https://webhook-error.example.com/test and project ${projectId}`,
);
}
// Verify trigger was disabled after 5 consecutive failures
const trigger = await prisma.trigger.findUnique({
where: { id: triggerId },
});
expect(trigger?.status).toBe(JobConfigState.INACTIVE);
// Verify lastFailingExecutionId was stored in action config
const updatedAction = await prisma.action.findUnique({
where: { id: actionId },
});
const config = updatedAction?.config as any;
expect(config.lastFailingExecutionId).toBeDefined();
expect(typeof config.lastFailingExecutionId).toBe("string");
// Encrypted request headers are preserved when lastFailingExecutionId is written.
expect(config.requestHeaders["x-secret-api-key"].value).toBe(
encryptedSecretHeaderValue,
);
expect(config.requestHeaders["x-secret-api-key"].value).not.toBe(
secretHeaderValue,
);
expect(decrypt(config.requestHeaders["x-secret-api-key"].value)).toBe(
secretHeaderValue,
);
});
it("clears the monitor-alert failure counter when auto-disabling the trigger", async () => {
const action = await prisma.action.findUnique({
where: { id: actionId },
});
if (!action) {
throw new Error("Action not found");
}
await prisma.action.update({
where: { id: actionId },
data: {
projectId,
type: "WEBHOOK",
config: {
...(action.config as WebhookActionConfigWithSecrets),
url: "https://webhook-error.example.com/test",
},
},
});
const failureKey = `automation-failures:${projectId}:${automationId}`;
await redis!.del(failureKey);
for (let i = 0; i < 5; i++) {
const webhookInput: WebhookInput = {
projectId,
automationId,
executionId: v4(),
payload: {
id: v4(),
timestamp: new Date(),
type: "monitor-alert",
apiVersion: "v1",
payload: {
monitorId,
projectId,
severity: "ALERT",
permalink: `https://cloud.langfuse.com/project/${projectId}/monitors/mon`,
timestamp: new Date(),
fromTimestamp: new Date(Date.now() - 5 * 60_000),
toTimestamp: new Date(),
message: { title: "High error rate", body: "errors > 100" },
view: "observations",
filters: [],
window: "5m",
},
},
};
await executeWebhook(webhookInput, { skipValidation: true });
}
const trigger = await prisma.trigger.findUnique({
where: { id: triggerId },
});
expect(trigger?.status).toBe(JobConfigState.INACTIVE);
expect(await redis!.get(failureKey)).toBeNull();
});
it("delivers a project notification envelope verbatim and completes the execution row", async () => {
const event = {
eventType: "blob-export-failed" as const,
severity: "ALERT" as const,
projectId,
projectName: "Test Project",
resourceId: projectId,
resourceName: "test-export-bucket",
message: "Blob storage export failed.",
url: `https://cloud.langfuse.com/project/${projectId}/settings`,
};
const executionId = v4();
const webhookInput: WebhookInput = {
projectId,
automationId,
executionId,
payload: {
id: executionId,
timestamp: new Date(),
type: "project-notification",
apiVersion: "v1",
event,
},
};
// dispatchProjectNotification creates a PENDING row per enqueued job.
await prisma.automationExecution.create({
data: {
id: executionId,
projectId,
automationId,
triggerId,
actionId,
status: ActionExecutionStatus.PENDING,
sourceId: event.resourceId,
input: event,
},
});
await executeWebhook(webhookInput, { skipValidation: true });
const requests = webhookServer.getReceivedRequests();
expect(requests).toHaveLength(1);
const body = JSON.parse(requests[0].body);
expect(body.type).toBe("project-notification");
expect(body.apiVersion).toBe("v1");
expect(body.id).toBe(executionId);
expect(body.event).toEqual(event);
// like the prompt-version path, the execution row is resolved to COMPLETED
const execution = await prisma.automationExecution.findUnique({
where: { id: executionId },
});
expect(execution?.status).toBe(ActionExecutionStatus.COMPLETED);
});
it("project-notification webhook: records ERROR rows on failure but never disables the trigger", async () => {
// Point the webhook action at the always-500 endpoint.
const action = await prisma.action.findUnique({
where: { id: actionId },
});
if (!action) throw new Error("Action not found");
await prisma.action.update({
where: { id: actionId },
data: {
config: {
...(action.config as WebhookActionConfigWithSecrets),
url: "https://webhook-error.example.com/test",
},
},
});
// Fail more times than the (prompt/monitor) disable threshold of 5.
for (let i = 0; i < 6; i++) {
const executionId = v4();
await prisma.automationExecution.create({
data: {
id: executionId,
projectId,
automationId,
triggerId,
actionId,
status: ActionExecutionStatus.PENDING,
sourceId: projectId,
input: {},
},
});
const webhookInput: WebhookInput = {
projectId,
automationId,
executionId,
payload: {
id: executionId,
timestamp: new Date(),
type: "project-notification",
apiVersion: "v1",
event: {
eventType: "blob-export-failed",
severity: "ALERT",
projectId,
projectName: "Test Project",
resourceId: projectId,
resourceName: "test-export-bucket",
message: "Blob storage export failed.",
},
},
};
await executeWebhook(webhookInput, { skipValidation: true });
const execution = await prisma.automationExecution.findUnique({
where: { id: executionId },
});
expect(execution?.status).toBe(ActionExecutionStatus.ERROR);
}
// Project-notification channels never auto-disable.
const trigger = await prisma.trigger.findUnique({
where: { id: triggerId },
});
expect(trigger?.status).toBe(JobConfigState.ACTIVE);
});
it("auto-disables despite a redis.del failure on the failure-side reset", async () => {
const action = await prisma.action.findUnique({
where: { id: actionId },
});
if (!action) {
throw new Error("Action not found");
}
await prisma.action.update({
where: { id: actionId },
data: {
projectId,
type: "WEBHOOK",
config: {
...(action.config as WebhookActionConfigWithSecrets),
url: "https://webhook-error.example.com/test",
},
},
});
const failureKey = `automation-failures:${projectId}:${automationId}`;
await redis!.del(failureKey);
const buildInput = (): WebhookInput => ({
projectId,
automationId,
executionId: v4(),
payload: {
id: v4(),
timestamp: new Date(),
type: "monitor-alert",
apiVersion: "v1",
payload: {
monitorId,
projectId,
severity: "ALERT",
permalink: `https://cloud.langfuse.com/project/${projectId}/monitors/mon`,
timestamp: new Date(),
fromTimestamp: new Date(Date.now() - 5 * 60_000),
toTimestamp: new Date(),
message: { title: "High error rate", body: "errors > 100" },
view: "observations",
filters: [],
window: "5m",
},
},
});
for (let i = 0; i < 4; i++) {
await executeWebhook(buildInput(), { skipValidation: true });
}
const delSpy = vi
.spyOn(redis!, "del")
.mockRejectedValue(new Error("READONLY during failover"));
try {
await executeWebhook(buildInput(), { skipValidation: true });
} finally {
delSpy.mockRestore();
}
const trigger = await prisma.trigger.findUnique({
where: { id: triggerId },
});
expect(trigger?.status).toBe(JobConfigState.INACTIVE);
});
it("should execute webhook with secret headers correctly", async () => {
// Get the full prompt for the payload
const fullPrompt = await prisma.prompt.findUnique({
where: { id: promptId },
});
// Update action config to include both public and secret headers
const action = await prisma.action.findUnique({
where: { id: actionId },
});
if (!action) {
throw new Error("Action not found");
}
const updatedConfig = {
...(action.config as WebhookActionConfigWithSecrets),
requestHeaders: {
"x-public-header": { secret: false, value: "public-value" },
"x-secret-api-key": {
secret: true,
value: encrypt("secret-api-key-value"),
},
"x-secret-token": {
secret: true,
value: encrypt("bearer-token-value"),
},
},
displayHeaders: {
"x-secret-api-key": { secret: true, value: "secr...alue" },
"x-secret-token": { secret: true, value: "bear...alue" },
},
};
await prisma.action.update({
where: { id: actionId },
data: {
config: updatedConfig,
},
});
const webhookInput: WebhookInput = {
projectId,
automationId,
executionId,
payload: {
prompt: PromptDomainSchema.parse(fullPrompt),
action: "created",
type: "prompt-version",
user: {
id: "user-123",
name: "Test User",
email: "[email protected]",
},
},
};
await prisma.automationExecution.create({
data: {
id: executionId,
projectId,
triggerId,
automationId,
actionId,
status: ActionExecutionStatus.PENDING,
sourceId: webhookInput.executionId,
input: webhookInput,
},
});
await executeWebhook(webhookInput, { skipValidation: true });
// Verify webhook request was made with decrypted secret headers
const requests = webhookServer.getReceivedRequests();
expect(requests).toHaveLength(1);
const request = requests[0];
expect(request.url).toBe("https://webhook.example.com/test");
expect(request.method).toBe("POST");
// Verify public header is present
expect(request.headers["x-public-header"]).toBe("public-value");
// Verify secret headers are present and decrypted
expect(request.headers["x-secret-api-key"]).toBe("secret-api-key-value");
expect(request.headers["x-secret-token"]).toBe("bearer-token-value");
// Verify signature is present
expect(request.headers["x-langfuse-signature"]).toMatch(
/^t=\d+,v1=[a-f0-9]+$/,
);
// Verify database execution record was updated
const execution = await prisma.automationExecution.findUnique({
where: { id: executionId },
});
expect(execution?.status).toBe(ActionExecutionStatus.COMPLETED);
});
it("should handle webhook with only secret headers", async () => {
// Get the full prompt for the payload
const fullPrompt = await prisma.prompt.findUnique({
where: { id: promptId },
});
// Update action config to include only secret headers
const action = await prisma.action.findUnique({
where: { id: actionId },
});
if (!action) {
throw new Error("Action not found");
}
const updatedConfig = {
...(action.config as WebhookActionConfigWithSecrets),
requestHeaders: {
authorization: {
secret: true,
value: encrypt("Bearer secret-token-12345"),
},
"x-api-key": { secret: true, value: encrypt("api-key-67890") },
},
displayHeaders: {
authorization: { secret: true, value: "Bear***12345" },
"x-api-key": { secret: true, value: "api-***67890" },
},
};
await prisma.action.update({
where: { id: actionId },
data: {
config: updatedConfig,
},
});
const newExecutionId = v4();
const webhookInput: WebhookInput = {
projectId,
automationId,
executionId: newExecutionId,
payload: {
prompt: PromptDomainSchema.parse(fullPrompt),
action: "created",
type: "prompt-version",
user: {
id: "user-123",
name: "Test User",
email: "[email protected]",
},
},
};
await prisma.automationExecution.create({
data: {
id: newExecutionId,
projectId,
triggerId,
automationId,
actionId,
status: ActionExecutionStatus.PENDING,
sourceId: webhookInput.executionId,
input: webhookInput,
},
});
await executeWebhook(webhookInput, { skipValidation: true });
// Verify webhook request was made with decrypted secret headers
const requests = webhookServer.getReceivedRequests();
expect(requests).toHaveLength(1); // This test only
const request = requests[0]; // Get the request
expect(request.url).toBe("https://webhook.example.com/test");
// Verify secret headers are present and decrypted
expect(request.headers["authorization"]).toBe(
"Bearer secret-token-12345",
);
expect(request.headers["x-api-key"]).toBe("api-key-67890");
// Verify database execution record was updated
const execution = await prisma.automationExecution.findUnique({
where: { id: newExecutionId },
});
expect(execution?.status).toBe(ActionExecutionStatus.COMPLETED);
});
it("should handle webhook with only legacy public headers", async () => {
// Get the full prompt for the payload
const fullPrompt = await prisma.prompt.findUnique({
where: { id: promptId },
});
// Update action config to include only public headers
const action = await prisma.action.findUnique({
where: { id: actionId },
});
if (!action) {
throw new Error("Action not found");
}
const updatedConfig = {
...(action.config as WebhookActionConfigWithSecrets),
headers: {
"x-public-header-1": "public-value-1",
"x-public-header-2": "public-value-2",
},
secretHeaderKeys: [], // No secret headers
displayHeaders: {},
};
await prisma.action.update({
where: { id: actionId },
data: {
config: updatedConfig,
},
});
const newExecutionId = v4();
const webhookInput: WebhookInput = {
projectId,
automationId,
executionId: newExecutionId,
payload: {
prompt: PromptDomainSchema.parse(fullPrompt),
action: "created",
type: "prompt-version",
user: {
id: "user-123",
name: "Test User",
email: "[email protected]",
},
},
};
await prisma.automationExecution.create({
data: {
id: newExecutionId,
projectId,
triggerId,
automationId,
actionId,
status: ActionExecutionStatus.PENDING,
sourceId: webhookInput.executionId,
input: webhookInput,
},
});
await executeWebhook(webhookInput, { skipValidation: true });
// Verify webhook request was made with public headers
const requests = webhookServer.getReceivedRequests();
expect(requests).toHaveLength(1); // This test only
const request = requests[0]; // Get the request
expect(request.url).toBe("https://webhook.example.com/test");
// Verify public headers are present
expect(request.headers["x-public-header-1"]).toBe("public-value-1");
expect(request.headers["x-public-header-2"]).toBe("public-value-2");
// Verify database execution record was updated
const execution = await prisma.automationExecution.findUnique({
where: { id: newExecutionId },
});
expect(execution?.status).toBe(ActionExecutionStatus.COMPLETED);
});
it("should handle webhook with legacy public headers + new public and secret headers", async () => {
// Get the full prompt for the payload
const fullPrompt = await prisma.prompt.findUnique({
where: { id: promptId },
});
// Update action config to include only public headers
const action = await prisma.action.findUnique({
where: { id: actionId },
});
if (!action) {
throw new Error("Action not found");
}
const updatedConfig = {
...(action.config as WebhookActionConfigWithSecrets),
headers: {
"x-public-header-1": "public-value-1",
"x-public-header-2": "public-value-2",
},
requestHeaders: {
"x-public-header-3": { secret: false, value: "public-value-3" },
"x-secret-header-4": {
secret: true,
value: encrypt("secret-value-4"),
},
"x-public-header-1": { secret: false, value: "public-value-5" },
},
displayHeaders: {},
};
await prisma.action.update({
where: { id: actionId },
data: {
config: updatedConfig,
},
});
const newExecutionId = v4();
const webhookInput: WebhookInput = {
projectId,
automationId,
executionId: newExecutionId,
payload: {
prompt: PromptDomainSchema.parse(fullPrompt),
action: "created",
type: "prompt-version",
user: {
id: "user-123",
name: "Test User",
email: "[email protected]",
},
},
};
await prisma.automationExecution.create({
data: {
id: newExecutionId,
projectId,
triggerId,
automationId,
actionId,
status: ActionExecutionStatus.PENDING,
sourceId: webhookInput.executionId,
input: webhookInput,
},
});
await executeWebhook(webhookInput, { skipValidation: true });
// Verify webhook request was made with public headers
const requests = webhookServer.getReceivedRequests();
expect(requests).toHaveLength(1); // This test only
const request = requests[0]; // Get the request
expect(request.url).toBe("https://webhook.example.com/test");
// Verify public headers are present
expect(request.headers["x-public-header-1"]).toBe("public-value-5");
expect(request.headers["x-public-header-2"]).toBe("public-value-2");
expect(request.headers["x-public-header-3"]).toBe("public-value-3");
expect(request.headers["x-secret-header-4"]).toBe("secret-value-4");
// Verify database execution record was updated
const execution = await prisma.automationExecution.findUnique({
where: { id: newExecutionId },
});
expect(execution?.status).toBe(ActionExecutionStatus.COMPLETED);
});
it("should handle decryption failure gracefully", async () => {
// Get the full prompt for the payload
const fullPrompt = await prisma.prompt.findUnique({
where: { id: promptId },
});
// Update action config with invalid encrypted header
const action = await prisma.action.findUnique({
where: { id: actionId },
});
if (!action) {
throw new Error("Action not found");
}
const updatedConfig = {
...(action.config as WebhookActionConfigWithSecrets),
requestHeaders: {
"x-valid-header": { secret: false, value: "valid-public-value" },
"x-invalid-secret": {
secret: true,
value: "invalid-encrypted-value", // This is not properly encrypted
},
},
displayHeaders: {
"x-invalid-secret": { secret: true, value: "inva...alue" },
},
};
await prisma.action.update({
where: { id: actionId },
data: {
config: updatedConfig,
},
});
const newExecutionId = v4();
const webhookInput: WebhookInput = {
projectId,
automationId,
executionId: newExecutionId,
payload: {
prompt: PromptDomainSchema.parse(fullPrompt),
action: "created",
type: "prompt-version",
user: {
id: "user-123",
name: "Test User",
email: "[email protected]",
},
},
};
await prisma.automationExecution.create({
data: {
id: newExecutionId,
projectId,
triggerId,
automationId,
actionId,
status: ActionExecutionStatus.PENDING,
sourceId: webhookInput.executionId,
input: webhookInput,
},
});
await executeWebhook(webhookInput, { skipValidation: true });
// Verify webhook request was made with valid headers only
const requests = webhookServer.getReceivedRequests();
expect(requests).toHaveLength(1); // This test only
const request = requests[0]; // Get the request
expect(request.url).toBe("https://webhook.example.com/test");
// Verify valid header is present
expect(request.headers["x-valid-header"]).toBe("valid-public-value");
// Verify invalid secret header is not present (skipped due to decryption failure)
expect(request.headers["x-invalid-secret"]).toBeUndefined();
// Verify database execution record was updated (should still succeed)
const execution = await prisma.automationExecution.findUnique({
where: { id: newExecutionId },
});
expect(execution?.status).toBe(ActionExecutionStatus.COMPLETED);
});
it("should handle missing automation gracefully without throwing error", async () => {
const fullPrompt = await prisma.prompt.findUnique({
where: { id: promptId },
});
const executionId = v4();
const nonExistentAutomationId = v4(); // Use a random ID that doesn't exist
const webhookInput: WebhookInput = {
projectId,
automationId: nonExistentAutomationId,
executionId,
payload: {
prompt: PromptDomainSchema.parse(fullPrompt),
action: "created",
type: "prompt-version",
user: {
id: "user-123",
name: "Test User",
email: "[email protected]",
},
},
};
// Should not throw an error, but return gracefully
await expect(
executeWebhook(webhookInput, { skipValidation: true }),
).resolves.toBeUndefined();
// Verify that no execution record was created since automation doesn't exist
const execution = await prisma.automationExecution.findUnique({
where: { id: executionId },
});
expect(execution).toBeNull();
});
it("should reset failure count correctly with lastFailingExecutionId", async () => {
const fullPrompt = await prisma.prompt.findUnique({
where: { id: promptId },
});
const action = await prisma.action.findUnique({
where: { id: actionId },
});
if (!action) {
throw new Error("Action not found");
}
// Set a lastFailingExecutionId in the action config
const lastFailingExecutionId = v4();
await prisma.action.update({
where: { id: actionId },
data: {
config: {
...(action.config as WebhookActionConfigWithSecrets),
lastFailingExecutionId,
},
},
});
// Create the execution that matches the lastFailingExecutionId
await prisma.automationExecution.create({
data: {
id: lastFailingExecutionId,
projectId,
triggerId,
automationId,
actionId,
status: ActionExecutionStatus.ERROR,
sourceId: v4(),
input: { test: "old failing execution" },
error: "Old failure",
createdAt: new Date(Date.now() - 60000), // 1 minute ago
},
});
// Import the function to test it directly
const { getConsecutiveAutomationFailures } =
await import("@langfuse/shared/src/server");
// Check that consecutive failures is 0 since there are no executions after the lastFailingExecutionId
const failures = await getConsecutiveAutomationFailures({
automationId,
projectId,
});
expect(failures).toBe(0);
});
it(
"should reset failure count when webhook is re-enabled after being disabled",
{ timeout: 20000 },
async () => {
const fullPrompt = await prisma.prompt.findUnique({
where: { id: promptId },
});
const action = await getActionByIdWithSecrets({
projectId,
actionId,
});
if (!action) {
throw new Error("Action not found");
}
// Update action to point to error endpoint
await prisma.action.update({
where: { id: actionId },
data: {
config: {
...(action.config as WebhookActionConfigWithSecrets),
url: "https://webhook-error.example.com/test",
},
},
});
const executionIds: string[] = [];
// Execute webhook 5 times to trigger disable
for (let i = 0; i < 5; i++) {
const executionId = v4();
executionIds.push(executionId);
await prisma.automationExecution.create({
data: {
id: executionId,
projectId,
triggerId,
automationId,
actionId,
status: ActionExecutionStatus.PENDING,
sourceId: v4(),
input: {
promptName: "test-prompt",
promptVersion: 1,
action: "created",
type: "prompt-version",
},
},
});
const webhookInput: WebhookInput = {
projectId,
automationId,
executionId,
payload: {
prompt: PromptDomainSchema.parse(fullPrompt),
action: "created",
type: "prompt-version",
},
};
await executeWebhook(webhookInput, { skipValidation: true });
}
// Verify trigger was disabled and lastFailingExecutionId was stored
const trigger = await prisma.trigger.findUnique({
where: { id: triggerId },
});
expect(trigger?.status).toBe(JobConfigState.INACTIVE);
const actionAfterDisable = await prisma.action.findUnique({
where: { id: actionId },
});
const configAfterDisable = actionAfterDisable?.config as any;
expect(configAfterDisable.lastFailingExecutionId).toBe(executionIds[4]);
// Re-enable the trigger (simulates user action)
await prisma.trigger.update({
where: { id: triggerId },
data: { status: JobConfigState.ACTIVE },
});
// Create 5 more failing executions to trigger disable again
const newExecutionIds: string[] = [];
for (let i = 0; i < 5; i++) {
const newExecutionId = v4();
newExecutionIds.push(newExecutionId);
await prisma.automationExecution.create({
data: {
id: newExecutionId,
projectId,
triggerId,
automationId,
actionId,
status: ActionExecutionStatus.PENDING,
sourceId: v4(),
input: {
promptName: "test-prompt",
promptVersion: 1,
action: "created",
type: "prompt-version",
},
},
});
const newWebhookInput: WebhookInput = {
projectId,
automationId,
executionId: newExecutionId,
payload: {
prompt: PromptDomainSchema.parse(fullPrompt),
action: "created",
type: "prompt-version",
},
};
await executeWebhook(newWebhookInput, { skipValidation: true });
}
// Verify trigger was disabled again and lastFailingExecutionId was updated
const triggerAfterSecondDisable = await prisma.trigger.findUnique({
where: { id: triggerId },
});
expect(triggerAfterSecondDisable?.status).toBe(JobConfigState.INACTIVE);
const actionAfterSecondDisable = await prisma.action.findUnique({
where: { id: actionId },
});
const configAfterSecondDisable =
actionAfterSecondDisable?.config as any;
expect(configAfterSecondDisable.lastFailingExecutionId).toBe(
newExecutionIds[4],
);
},
);
it("should include user info in webhook payload when provided", async () => {
const fullPrompt = await prisma.prompt.findUnique({
where: { id: promptId },
});
const testUser = {
id: "user-123",
name: "Test User",
email: "[email protected]",
};
const webhookInput: WebhookInput = {
projectId,
automationId,
executionId,
payload: {
prompt: PromptDomainSchema.parse(fullPrompt),
action: "created",
type: "prompt-version",
user: testUser,
},
};
await prisma.automationExecution.create({
data: {
id: executionId,
projectId,
triggerId,
automationId,
actionId,
status: ActionExecutionStatus.PENDING,
sourceId: webhookInput.executionId,
input: webhookInput,
},
});
await executeWebhook(webhookInput, { skipValidation: true });
const requests = webhookServer.getReceivedRequests();
expect(requests).toHaveLength(1);
const payload = JSON.parse(requests[0].body);
expect(payload.user).toEqual({
name: testUser.name,
email: testUser.email,
});
expect(payload.user.id).toBeUndefined();
// Verify prompt is still the last field
const payloadKeys = Object.keys(payload);
expect(payloadKeys[payloadKeys.length - 1]).toBe("prompt");
});
it("should omit user field from webhook payload when not provided", async () => {
const fullPrompt = await prisma.prompt.findUnique({
where: { id: promptId },
});
const webhookInput: WebhookInput = {
projectId,
automationId,
executionId,
payload: {
prompt: PromptDomainSchema.parse(fullPrompt),
action: "created",
type: "prompt-version",
},
};
await prisma.automationExecution.create({
data: {
id: executionId,
projectId,
triggerId,
automationId,
actionId,
status: ActionExecutionStatus.PENDING,
sourceId: webhookInput.executionId,
input: webhookInput,
},
});
await executeWebhook(webhookInput, { skipValidation: true });
const requests = webhookServer.getReceivedRequests();
expect(requests).toHaveLength(1);
const payload = JSON.parse(requests[0].body);
expect(payload.user).toBeUndefined();
});
it("should include user info in GitHub dispatch payload when provided", async () => {
const fullPrompt = await prisma.prompt.findUnique({
where: { id: promptId },
});
const testUser = {
id: "user-456",
name: "GitHub User",
email: "[email protected]",
};
// Create a GitHub dispatch action
const ghActionId = v4();
await prisma.action.create({
data: {
id: ghActionId,
projectId,
type: "GITHUB_DISPATCH",
config: {
type: "GITHUB_DISPATCH",
url: "https://webhook.example.com/dispatches",
eventType: "prompt-update",
githubToken: encrypt("ghp_test_token"),
displayGitHubToken: "ghp_...n",
},
},
});
// Create automation linking trigger and GitHub dispatch action
const ghAutomationId = v4();
await prisma.automation.create({
data: {
id: ghAutomationId,
projectId,
triggerId,
actionId: ghActionId,
name: "GitHub Dispatch Automation",
},
});
const ghExecutionId = v4();
await prisma.automationExecution.create({
data: {
id: ghExecutionId,
projectId,
triggerId,
automationId: ghAutomationId,
actionId: ghActionId,
status: ActionExecutionStatus.PENDING,
sourceId: promptId,
input: {},
},
});
const webhookInput: WebhookInput = {
projectId,
automationId: ghAutomationId,
executionId: ghExecutionId,
payload: {
prompt: PromptDomainSchema.parse(fullPrompt),
action: "created",
type: "prompt-version",
user: testUser,
},
};
await executeWebhook(webhookInput, { skipValidation: true });
const requests = webhookServer.getReceivedRequests();
expect(requests).toHaveLength(1);
const payload = JSON.parse(requests[0].body);
expect(payload.event_type).toBe("prompt-update");
expect(payload.client_payload.user).toEqual({
name: testUser.name,
email: testUser.email,
});
expect(payload.client_payload.user.id).toBeUndefined();
// Verify prompt is still the last field in client_payload
const clientPayloadKeys = Object.keys(payload.client_payload);
expect(clientPayloadKeys[clientPayloadKeys.length - 1]).toBe("prompt");
});
it("applies the webhook host allowlist to webhook delivery and not GitHub dispatch", async () => {
const resolve4 = vi
.spyOn(dns, "resolve4")
.mockResolvedValue(["93.184.216.34"]);
const resolve6 = vi
.spyOn(dns, "resolve6")
.mockRejectedValue(
Object.assign(new Error("ENODATA"), { code: "ENODATA" }),
);
const lookup = vi
.spyOn(dns, "lookup")
.mockResolvedValue([{ address: "93.184.216.34", family: 4 }] as never);
outboundWhitelistBuilders.webhookWhitelistFromEnv.mockReturnValue({
hosts: [],
ips: [],
ip_ranges: [],
allowedHosts: ["hooks.example.com"],
});
outboundWhitelistBuilders.whitelistFromEnv.mockReturnValue({
hosts: [],
ips: [],
ip_ranges: [],
});
try {
const fullPrompt = await prisma.prompt.findUnique({
where: { id: promptId },
});
const promptPayload = {
prompt: PromptDomainSchema.parse(fullPrompt),
action: "created" as const,
type: "prompt-version" as const,
};
const webhookInput: WebhookInput = {
projectId,
automationId,
executionId,
payload: promptPayload,
};
await prisma.automationExecution.create({
data: {
id: executionId,
projectId,
triggerId,
automationId,
actionId,
status: ActionExecutionStatus.PENDING,
sourceId: promptId,
input: webhookInput,
},
});
await executeWebhook(webhookInput);
const webhookExecution = await prisma.automationExecution.findUnique({
where: { id: executionId },
});
expect(webhookExecution?.status).toBe(ActionExecutionStatus.ERROR);
expect(webhookExecution?.error).toContain(
"Host is not in the outbound URL allowlist",
);
expect(webhookServer.getReceivedRequests()).toHaveLength(0);
const ghActionId = v4();
await prisma.action.create({
data: {
id: ghActionId,
projectId,
type: "GITHUB_DISPATCH",
config: {
type: "GITHUB_DISPATCH",
url: "https://webhook.example.com/dispatches",
eventType: "prompt-update",
githubToken: encrypt("ghp_test_token"),
displayGitHubToken: "ghp_...n",
},
},
});
const ghAutomationId = v4();
await prisma.automation.create({
data: {
id: ghAutomationId,
projectId,
triggerId,
actionId: ghActionId,
name: "GitHub Dispatch Automation",
},
});
const ghExecutionId = v4();
await prisma.automationExecution.create({
data: {
id: ghExecutionId,
projectId,
triggerId,
automationId: ghAutomationId,
actionId: ghActionId,
status: ActionExecutionStatus.PENDING,
sourceId: promptId,
input: {},
},
});
await executeWebhook({
projectId,
automationId: ghAutomationId,
executionId: ghExecutionId,
payload: promptPayload,
});
expect(webhookServer.getReceivedRequests()).toHaveLength(1);
const dispatchExecution = await prisma.automationExecution.findUnique({
where: { id: ghExecutionId },
});
expect(dispatchExecution?.status).toBe(ActionExecutionStatus.COMPLETED);
} finally {
outboundWhitelistBuilders.webhookWhitelistFromEnv.mockImplementation(
outboundWhitelistBuilders.realWebhookWhitelistFromEnv,
);
outboundWhitelistBuilders.whitelistFromEnv.mockImplementation(
outboundWhitelistBuilders.realWhitelistFromEnv,
);
resolve4.mockRestore();
resolve6.mockRestore();
lookup.mockRestore();
}
});
it("should truncate oversized GitHub dispatch prompt fields", async () => {
const fullPrompt = await prisma.prompt.findUnique({
where: { id: promptId },
});
if (!fullPrompt) {
throw new Error("Prompt not found");
}
const testUser = {
id: "user-456",
name: "GitHub User",
email: "[email protected]",
};
const ghActionId = v4();
await prisma.action.create({
data: {
id: ghActionId,
projectId,
type: "GITHUB_DISPATCH",
config: {
type: "GITHUB_DISPATCH",
url: "https://webhook.example.com/dispatches",
eventType: "prompt-update",
githubToken: encrypt("ghp_test_token"),
displayGitHubToken: "ghp_...n",
},
},
});
const ghAutomationId = v4();
await prisma.automation.create({
data: {
id: ghAutomationId,
projectId,
triggerId,
actionId: ghActionId,
name: "GitHub Dispatch Automation",
},
});
const ghExecutionId = v4();
await prisma.automationExecution.create({
data: {
id: ghExecutionId,
projectId,
triggerId,
automationId: ghAutomationId,
actionId: ghActionId,
status: ActionExecutionStatus.PENDING,
sourceId: promptId,
input: {},
},
});
const webhookInput: WebhookInput = {
projectId,
automationId: ghAutomationId,
executionId: ghExecutionId,
payload: {
prompt: PromptDomainSchema.parse({
...fullPrompt,
prompt: {
content: "a".repeat(GITHUB_REPOSITORY_DISPATCH_MAX_PAYLOAD_BYTES),
},
config: {
schema: {
content: "b".repeat(
GITHUB_REPOSITORY_DISPATCH_MAX_PAYLOAD_BYTES,
),
},
},
}),
action: "created",
type: "prompt-version",
user: testUser,
},
};
await executeWebhook(webhookInput, { skipValidation: true });
const requests = webhookServer.getReceivedRequests();
expect(requests).toHaveLength(1);
expect(Buffer.byteLength(requests[0].body, "utf8")).toBeLessThan(
GITHUB_REPOSITORY_DISPATCH_MAX_PAYLOAD_BYTES,
);
const payload = JSON.parse(requests[0].body);
expect(payload.event_type).toBe("prompt-update");
expect(Object.keys(payload.client_payload)).toHaveLength(8);
expect(payload.client_payload.truncation).toEqual({
payloadTruncated: true,
truncatedFields: ["prompt.prompt", "prompt.config"],
});
expect(payload.client_payload.prompt.prompt).toBe(
GITHUB_REPOSITORY_DISPATCH_TRUNCATION_MARKER,
);
expect(payload.client_payload.prompt.config).toEqual({});
const clientPayloadKeys = Object.keys(payload.client_payload);
expect(clientPayloadKeys[clientPayloadKeys.length - 1]).toBe("prompt");
});
it("should truncate oversized GitHub dispatch monitor-alert filters", async () => {
const ghActionId = v4();
await prisma.action.create({
data: {
id: ghActionId,
projectId,
type: "GITHUB_DISPATCH",
config: {
type: "GITHUB_DISPATCH",
url: "https://webhook.example.com/dispatches",
eventType: "monitor-alert",
githubToken: encrypt("ghp_test_token"),
displayGitHubToken: "ghp_...n",
},
},
});
const ghAutomationId = v4();
await prisma.automation.create({
data: {
id: ghAutomationId,
projectId,
triggerId,
actionId: ghActionId,
name: "GitHub Dispatch Monitor Automation",
},
});
const ghExecutionId = v4();
await prisma.automationExecution.create({
data: {
id: ghExecutionId,
projectId,
triggerId,
automationId: ghAutomationId,
actionId: ghActionId,
status: ActionExecutionStatus.PENDING,
sourceId: v4(),
input: {},
},
});
const webhookInput: WebhookInput = {
projectId,
automationId: ghAutomationId,
executionId: ghExecutionId,
payload: {
id: v4(),
timestamp: new Date(),
type: "monitor-alert",
apiVersion: "v1",
payload: {
monitorId,
projectId,
severity: "ALERT",
permalink: `https://cloud.langfuse.com/project/${projectId}/monitors/mon`,
timestamp: new Date(),
fromTimestamp: new Date(Date.now() - 5 * 60_000),
toTimestamp: new Date(),
message: { title: "High error rate", body: "errors > 100" },
view: "observations",
filters: [
{
column: "metadata.user_id",
operator: "any of",
value: [
"x".repeat(
GITHUB_REPOSITORY_DISPATCH_MAX_PAYLOAD_BYTES + 1000,
),
],
type: "stringOptions",
},
],
window: "5m",
},
},
};
await executeWebhook(webhookInput, { skipValidation: true });
const requests = webhookServer.getReceivedRequests();
expect(requests).toHaveLength(1);
expect(Buffer.byteLength(requests[0].body, "utf8")).toBeLessThan(
GITHUB_REPOSITORY_DISPATCH_MAX_PAYLOAD_BYTES,
);
const payload = JSON.parse(requests[0].body);
expect(payload.event_type).toBe("monitor-alert");
expect(payload.client_payload.payload.filters).toEqual([]);
expect(payload.client_payload.truncation).toEqual({
payloadTruncated: true,
truncatedFields: ["payload.filters", "payload.message.body"],
});
});
});
describe("stale monitor alerts", () => {
const alertedAt = new Date("2026-05-27T14:00:00.000Z");
const buildMonitorAlertInput = (alertAlertedAt?: Date) =>
({
projectId,
automationId,
executionId: v4(),
payload: {
id: v4(),
timestamp: new Date(),
type: "monitor-alert",
apiVersion: "v1",
payload: {
monitorId,
projectId,
severity: "ALERT",
timestamp: new Date(),
fromTimestamp: new Date(Date.now() - 5 * 60_000),
toTimestamp: new Date(),
message: { title: "High error rate", body: "errors > 100" },
view: "observations",
filters: [],
window: "5m",
alertedAt: alertAlertedAt,
},
},
}) satisfies WebhookInput;
const useErrorEndpoint = async () => {
const action = await prisma.action.findUniqueOrThrow({
where: { id: actionId },
});
await prisma.action.update({
where: { id: actionId },
data: {
config: {
...(action.config as WebhookActionConfigWithSecrets),
url: "https://webhook-error.example.com/test",
},
},
});
};
it("drops the alert without counting a failure once the monitor emitted a newer alert", async () => {
await prisma.monitor.update({
where: { id: monitorId },
data: { alertedAt: new Date(alertedAt.getTime() + 1) },
});
await useErrorEndpoint();
const failureKey = `automation-failures:${projectId}:${automationId}`;
await redis!.del(failureKey);
await executeWebhook(buildMonitorAlertInput(alertedAt), {
skipValidation: true,
});
expect(webhookServer.getReceivedRequests()).toHaveLength(0);
expect(await redis!.get(failureKey)).toBeNull();
});
it("drops the alert when its monitor was deleted", async () => {
await prisma.monitor.delete({ where: { id: monitorId } });
await executeWebhook(buildMonitorAlertInput(alertedAt), {
skipValidation: true,
});
expect(webhookServer.getReceivedRequests()).toHaveLength(0);
});
it("drops the alert for every action type, not only webhooks", async () => {
await prisma.monitor.update({
where: { id: monitorId },
data: { alertedAt: new Date(alertedAt.getTime() + 1) },
});
await prisma.action.update({
where: { id: actionId },
data: {
type: "GITHUB_DISPATCH",
config: {
type: "GITHUB_DISPATCH",
url: "https://webhook.example.com/dispatches",
eventType: "monitor-alert",
githubToken: encrypt("ghp_test_token"),
displayGitHubToken: "ghp_...n",
},
},
});
await executeWebhook(buildMonitorAlertInput(alertedAt), {
skipValidation: true,
});
expect(webhookServer.getReceivedRequests()).toHaveLength(0);
});
it.each([
// The processor publishes before it saves alertedAt.
["before the monitor saved it", new Date(0)],
["after the monitor saved it", alertedAt],
])("delivers the alert %s", async (_, monitorAlertedAt) => {
await prisma.monitor.update({
where: { id: monitorId },
data: { alertedAt: monitorAlertedAt },
});
await executeWebhook(buildMonitorAlertInput(alertedAt), {
skipValidation: true,
});
expect(webhookServer.getReceivedRequests()).toHaveLength(1);
});
it("delivers the alert when the monitor changed severity without emitting an alert", async () => {
await prisma.monitor.update({
where: { id: monitorId },
data: {
severityChangedAt: new Date(alertedAt.getTime() + 1),
alertedAt,
},
});
await executeWebhook(buildMonitorAlertInput(alertedAt), {
skipValidation: true,
});
expect(webhookServer.getReceivedRequests()).toHaveLength(1);
});
it("delivers alerts queued without alertedAt", async () => {
await prisma.monitor.update({
where: { id: monitorId },
data: { alertedAt: new Date() },
});
await executeWebhook(buildMonitorAlertInput(), {
skipValidation: true,
});
expect(webhookServer.getReceivedRequests()).toHaveLength(1);
});
});
describe("PagerDuty action", () => {
const routingKey = "R0abcdefghijklmnopqrstuvwxyz9xYz";
const setPagerDutyConfig = (region: "US" | "EU") =>
prisma.action.update({
where: { id: actionId },
data: {
type: "PAGERDUTY",
config: {
type: "PAGERDUTY",
routingKey: encrypt(routingKey),
displayRoutingKey: "R0ab...9xYz",
region,
},
},
});
const buildMonitorInput = (
severity: "ALERT" | "WARNING" | "OK" | "NO_DATA",
): WebhookInput => ({
projectId,
automationId,
executionId: v4(),
payload: {
id: v4(),
timestamp: new Date(),
type: "monitor-alert",
apiVersion: "v1",
payload: {
monitorId,
projectId,
severity,
permalink: `https://cloud.langfuse.com/project/${projectId}/alerts/${monitorId}`,
timestamp: new Date("2026-05-27T14:32:00.000Z"),
fromTimestamp: new Date("2026-05-27T14:00:00.000Z"),
toTimestamp: new Date("2026-05-27T14:30:00.000Z"),
message: { title: "Hallucination rate breached", body: "0.42 > 0.3" },
view: "observations",
filters: [
{
column: "providedModelName",
type: "stringOptions",
operator: "any of",
value: ["gpt-4o"],
},
],
window: "30m",
monitorName: "Hallucination rate",
metric: { measure: "latency", aggregation: "p95" },
metricValue: 0.42,
thresholdOperator: "GT",
alertThreshold: 0.3,
warningThreshold: 0.2,
},
},
});
const pagerDutyRequests = () =>
webhookServer.getReceivedRequests().map((request) => ({
url: request.url,
body: JSON.parse(request.body),
}));
const lastPagerDutyRequest = () => pagerDutyRequests().at(-1)!;
beforeEach(async () => {
await setPagerDutyConfig("US");
await redis!.del(`automation-failures:${projectId}:${automationId}`);
});
it("triggers a critical alert keyed by the monitor with LLMOps details", async () => {
await executeWebhook(buildMonitorInput("ALERT"));
const [{ url, body }] = pagerDutyRequests();
const project = await prisma.project.findUniqueOrThrow({
where: { id: projectId },
});
expect(url).toBe("https://events.pagerduty.com/v2/enqueue");
expect(body).toMatchObject({
routing_key: routingKey,
event_action: "trigger",
dedup_key: `langfuse-${projectId}-${monitorId}`,
client: "Langfuse",
links: [
{
href: `https://cloud.langfuse.com/project/${projectId}/alerts/${monitorId}`,
},
],
payload: {
summary: "Hallucination rate breached",
source: `langfuse/${project.name}`,
severity: "critical",
timestamp: "2026-05-27T14:32:00.000Z",
class: "monitor-alert",
component: "observations",
custom_details: {
llmops: {
model_name: "gpt-4o",
project_name: project.name,
monitor_name: "Hallucination rate",
monitor_id: monitorId,
monitor_alert_start: "2026-05-27T14:00:00.000Z",
monitor_alert_end: "2026-05-27T14:30:00.000Z",
metric_value: 0.42,
threshold_value: 0.3,
},
langfuse: { window: "30m" },
},
},
});
});
it.each([
["WARNING", "warning"],
["NO_DATA", "error"],
] as const)(
"maps %s to PagerDuty severity %s",
async (severity, expected) => {
await executeWebhook(buildMonitorInput(severity));
expect(pagerDutyRequests()[0].body.payload.severity).toBe(expected);
},
);
it("re-triggers the monitor's alert under the same key on a severity change", async () => {
await executeWebhook(buildMonitorInput("WARNING"));
await executeWebhook(buildMonitorInput("ALERT"));
expect(
pagerDutyRequests().map(({ body }) => [
body.event_action,
body.dedup_key,
body.payload.severity,
]),
).toEqual([
["trigger", `langfuse-${projectId}-${monitorId}`, "warning"],
["trigger", `langfuse-${projectId}-${monitorId}`, "critical"],
]);
});
it("resolves the monitor's alert on recovery", async () => {
await executeWebhook(buildMonitorInput("OK"));
expect(pagerDutyRequests().map(({ body }) => body)).toEqual([
{
routing_key: routingKey,
event_action: "resolve",
dedup_key: `langfuse-${projectId}-${monitorId}`,
},
]);
});
it("sends to the EU host for EU routing keys", async () => {
await setPagerDutyConfig("EU");
await executeWebhook(buildMonitorInput("ALERT"));
expect(pagerDutyRequests()[0].url).toBe(
"https://events.eu.pagerduty.com/v2/enqueue",
);
});
it.each(["prompt", "skill"] as const)(
"sends %s changes as change events and completes the execution",
async (entity) => {
const fullPrompt = await prisma.prompt.findUniqueOrThrow({
where: { id: promptId },
});
const input: WebhookInput = {
projectId,
automationId,
executionId,
payload:
entity === "prompt"
? {
prompt: PromptDomainSchema.parse(fullPrompt),
action: "created",
type: "prompt-version",
}
: {
type: "skill-version",
action: "created",
skill: {
id: v4(),
projectId,
name: "test-skill",
version: 1,
description: "Test skill",
frontmatter: {},
files: [],
tags: [],
labels: ["latest"],
commitMessage: null,
createdBy: "test-user",
createdAt: new Date(),
updatedAt: new Date(),
},
},
};
await prisma.automationExecution.create({
data: {
id: executionId,
projectId,
triggerId,
automationId,
actionId,
status: ActionExecutionStatus.PENDING,
sourceId: executionId,
input: {},
},
});
await executeWebhook(input);
const { url, body } = lastPagerDutyRequest();
expect(url).toBe("https://events.pagerduty.com/v2/change/enqueue");
expect(body).toMatchObject({
routing_key: routingKey,
payload: {
summary: `Langfuse ${entity} "test-${entity}" version 1 created`,
custom_details: {
[`${entity}_name`]: `test-${entity}`,
[`${entity}_version`]: 1,
},
},
});
expect(body).not.toHaveProperty("event_action");
expect(body.links[0].href).toContain(
`/project/${projectId}/${entity}s/`,
);
expect(body.links[0].href).toContain("version=1");
const execution = await prisma.automationExecution.findUniqueOrThrow({
where: { id: executionId },
});
expect(execution.status).toBe(ActionExecutionStatus.COMPLETED);
},
);
it("always triggers project notifications keyed by event type and resource", async () => {
await prisma.automationExecution.create({
data: {
id: executionId,
projectId,
triggerId,
automationId,
actionId,
status: ActionExecutionStatus.PENDING,
sourceId: executionId,
input: {},
},
});
await executeWebhook({
projectId,
automationId,
executionId,
payload: {
id: executionId,
timestamp: new Date(),
type: "project-notification",
apiVersion: "v1",
event: {
eventType: "blob-export-failed",
severity: "ALERT",
projectId,
projectName: "Test Project",
resourceId: "res_1",
resourceName: "my-bucket",
message: "Blob storage export failed.",
},
},
});
expect(lastPagerDutyRequest().body).toMatchObject({
event_action: "trigger",
dedup_key: `langfuse-${projectId}-blob-export-failed-res_1`,
payload: { severity: "error", class: "blob-export-failed" },
});
});
it.each([429, 500])(
"retries a %i in-process, then counts a failure once retries run out",
async (status) => {
webhookServer.pagerDutyStatus = status;
await executeWebhook(buildMonitorInput("ALERT"));
expect(webhookServer.getReceivedRequests()).toHaveLength(4);
expect(
await redis!.get(`automation-failures:${projectId}:${automationId}`),
).toBe("1");
},
);
it("counts a rejected event as a failure without retrying", async () => {
webhookServer.pagerDutyStatus = 400;
await executeWebhook(buildMonitorInput("ALERT"));
expect(
await redis!.get(`automation-failures:${projectId}:${automationId}`),
).toBe("1");
});
it("records PagerDuty's status on a rejected prompt change", async () => {
webhookServer.pagerDutyStatus = 400;
const fullPrompt = await prisma.prompt.findUniqueOrThrow({
where: { id: promptId },
});
await prisma.automationExecution.create({
data: {
id: executionId,
projectId,
triggerId,
automationId,
actionId,
status: ActionExecutionStatus.PENDING,
sourceId: executionId,
input: {},
},
});
await executeWebhook({
projectId,
automationId,
executionId,
payload: {
prompt: PromptDomainSchema.parse(fullPrompt),
action: "created",
type: "prompt-version",
},
});
const execution = await prisma.automationExecution.findUniqueOrThrow({
where: { id: executionId },
});
expect(execution).toMatchObject({
status: ActionExecutionStatus.ERROR,
output: { httpStatus: 400 },
});
});
it("returns the routing key only on the execution read path", async () => {
const withSecrets = await getActionByIdWithSecrets({
projectId,
actionId,
});
const safe = await getActionById({ projectId, actionId });
const config = withSecrets?.config as { routingKey: string };
expect(decrypt(config.routingKey)).toBe(routingKey);
expect(safe?.config).toEqual({
type: "PAGERDUTY",
displayRoutingKey: "R0ab...9xYz",
region: "US",
});
expect(JSON.stringify(safe)).not.toContain(routingKey);
});
});
});
describe("getActionByIdWithSecrets legacy headers", () => {
it("sends legacy headers as secret and never falls back to a legacy value its requestHeaders entry overrides", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const action = await prisma.action.create({
data: {
id: v4(),
projectId,
type: "WEBHOOK",
config: {
type: "WEBHOOK",
url: "https://example.com/webhook",
headers: {
"x-api-key": "legacy-api-key",
Authorization: "Bearer stale-token",
},
requestHeaders: {
Authorization: { secret: true, value: "not-a-valid-ciphertext" },
},
apiVersion: { prompt: "v1" },
secretKey: encrypt("whsec"),
displaySecretKey: "lf-whsec_...",
},
},
});
const result = await getActionByIdWithSecrets({
projectId,
actionId: action.id,
});
expect(
(result?.config as WebhookActionConfigWithSecrets).requestHeaders,
).toEqual({ "x-api-key": { secret: true, value: "legacy-api-key" } });
});
});