🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import {
describe,
it,
expect,
beforeEach,
beforeAll,
afterAll,
vi,
} from "vitest";
import { v4 } from "uuid";
import {
ActionExecutionStatus,
JobConfigState,
PromptDomain,
SlackActionConfig,
} from "@langfuse/shared";
import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";
import { prisma } from "@langfuse/shared/src/db";
import { encrypt } from "@langfuse/shared/encryption";
import { executeWebhook } from "../queues/webhooks";
import type { WebhookInput } from "@langfuse/shared/src/server";
// Mock SlackService
vi.mock("@langfuse/shared/src/server", async () => {
const actual = await vi.importActual("@langfuse/shared/src/server");
return {
...actual,
SlackService: {
getInstance: vi.fn(),
},
getActionById: vi.fn(),
};
});
describe("Slack Processor", () => {
let projectId: string;
let triggerId: string;
let actionId: string;
let automationId: string;
let promptId: string;
let executionId: string;
let monitorId: string;
let mockSlackService: any;
beforeAll(async () => {
// Import mocked SlackService
const server = await import("@langfuse/shared/src/server");
const { SlackService } = server;
// getActionById delegates to the real impl so existing tests read real rows
const actual = await vi.importActual<any>("@langfuse/shared/src/server");
(server.getActionById as any).mockImplementation(actual.getActionById);
// Create mock service instance
mockSlackService = {
getWebClientForProject: vi.fn(),
sendMessage: vi.fn(),
};
// Setup the getInstance mock to return our mock service
(SlackService.getInstance as any).mockReturnValue(mockSlackService);
// Setup default mock implementations
mockSlackService.getWebClientForProject.mockResolvedValue({
chat: { postMessage: vi.fn() },
});
mockSlackService.sendMessage.mockResolvedValue({
messageTs: "1234567890.123456",
channel: "C123456",
});
});
beforeEach(async () => {
// Reset mocks
vi.clearAllMocks();
// clearAllMocks wipes call history but preserves the getActionById impl
const actual = await vi.importActual<any>("@langfuse/shared/src/server");
const server = await import("@langfuse/shared/src/server");
(server.getActionById as any).mockImplementation(actual.getActionById);
// Create test project
({ projectId } = await createOrgProjectAndApiKey());
// Create Slack integration
await prisma.slackIntegration.create({
data: {
projectId,
teamId: "T123456",
teamName: "Test Team",
botToken: encrypt("xoxb-test-token"),
botUserId: "U123456",
},
});
// monitor-alert jobs are only delivered while their monitor exists
({ id: monitorId } = await prisma.monitor.create({
data: {
projectId,
name: "Test monitor",
view: "OBSERVATIONS",
filters: [],
metric: { measure: "count", aggregation: "count" },
windowMs: 5n * 60_000n,
cadenceMs: 60_000n,
thresholdOperator: "GT",
alertThreshold: 100,
noData: { mode: "SHOW_NO_DATA" },
renotify: { mode: "OFF" },
schedulerBatchId: 0n,
},
}));
// Create test prompt
promptId = v4();
await prisma.prompt.create({
data: {
id: promptId,
projectId,
name: "test-prompt",
version: 1,
type: "text",
prompt: { text: "Hello {{name}}" },
createdBy: "test-user",
},
});
// Create test trigger
triggerId = v4();
await prisma.trigger.create({
data: {
id: triggerId,
projectId,
eventSource: "prompt",
eventActions: ["created"],
filter: [],
status: JobConfigState.ACTIVE,
},
});
// Create test Slack action
actionId = v4();
await prisma.action.create({
data: {
id: actionId,
projectId,
type: "SLACK",
config: {
type: "SLACK",
channelId: "C123456",
channelName: "general",
} as SlackActionConfig,
},
});
// Link trigger to action
automationId = v4();
await prisma.automation.create({
data: {
id: automationId,
projectId,
triggerId,
actionId,
name: "Test Slack Automation",
},
});
executionId = v4();
});
afterAll(async () => {
vi.restoreAllMocks();
});
describe("executeSlack function", () => {
it("should execute slack action successfully", async () => {
const { SlackService } = await import("@langfuse/shared/src/server");
// Get the full prompt for the payload
const fullPrompt = await prisma.prompt.findUnique({
where: { id: promptId },
});
const slackInput: WebhookInput = {
projectId,
automationId,
executionId,
payload: {
prompt: fullPrompt as PromptDomain,
action: "created",
type: "prompt-version",
user: {
id: "user-123",
name: "Test User",
email: "[email protected]",
},
},
};
await prisma.automationExecution.create({
data: {
id: executionId,
projectId,
triggerId,
automationId,
actionId,
status: ActionExecutionStatus.PENDING,
sourceId: slackInput.executionId,
input: slackInput,
},
});
await executeWebhook(slackInput);
// Verify SlackService was called correctly
expect(SlackService.getInstance).toHaveBeenCalled();
expect(mockSlackService.getWebClientForProject).toHaveBeenCalledWith(
projectId,
);
expect(mockSlackService.sendMessage).toHaveBeenCalledWith({
client: expect.any(Object),
channelId: "C123456",
blocks: expect.any(Array),
text: "Langfuse Notification",
});
// Verify database execution record was updated
const execution = await prisma.automationExecution.findUnique({
where: { id: executionId },
});
expect(execution?.status).toBe(ActionExecutionStatus.COMPLETED);
expect(execution?.output).toMatchObject({
channel: "C123456",
messageTs: "1234567890.123456",
});
expect(execution?.startedAt).toBeDefined();
expect(execution?.finishedAt).toBeDefined();
});
it("delivers prompt changes whose config is not a JSON object", async () => {
const fullPrompt = await prisma.prompt.findUniqueOrThrow({
where: { id: promptId },
});
const slackInput: WebhookInput = {
projectId,
automationId,
executionId,
payload: {
prompt: { ...(fullPrompt as PromptDomain), config: [] },
action: "created",
type: "prompt-version",
},
};
await prisma.automationExecution.create({
data: {
id: executionId,
projectId,
triggerId,
automationId,
actionId,
status: ActionExecutionStatus.PENDING,
sourceId: slackInput.executionId,
input: {},
},
});
await executeWebhook(slackInput);
expect(mockSlackService.sendMessage).toHaveBeenCalled();
const execution = await prisma.automationExecution.findUniqueOrThrow({
where: { id: executionId },
});
expect(execution.status).toBe(ActionExecutionStatus.COMPLETED);
});
it("should handle missing automation gracefully", async () => {
const nonExistentAutomationId = v4();
const slackInput: WebhookInput = {
projectId,
automationId: nonExistentAutomationId,
executionId,
payload: {
prompt: { id: promptId } as PromptDomain,
action: "created",
type: "prompt-version",
user: {
id: "user-123",
name: "Test User",
email: "[email protected]",
},
},
};
// Should not throw an error, but return gracefully
await expect(executeWebhook(slackInput)).resolves.toBeUndefined();
// Verify that no SlackService calls were made
const { SlackService } = await import("@langfuse/shared/src/server");
expect(SlackService.getInstance).not.toHaveBeenCalled();
expect(mockSlackService.getWebClientForProject).not.toHaveBeenCalled();
expect(mockSlackService.sendMessage).not.toHaveBeenCalled();
});
it("should use custom template when provided", async () => {
const { SlackService } = await import("@langfuse/shared/src/server");
// Update action to include custom message template
const customTemplate = [
{
type: "section",
text: {
type: "mrkdwn",
text: "Custom Slack template message",
},
},
];
await prisma.action.update({
where: { id: actionId },
data: {
config: {
type: "SLACK",
channelId: "C123456",
channelName: "general",
messageTemplate: JSON.stringify(customTemplate),
} as SlackActionConfig,
},
});
const fullPrompt = await prisma.prompt.findUnique({
where: { id: promptId },
});
const slackInput: WebhookInput = {
projectId,
automationId,
executionId,
payload: {
prompt: fullPrompt as any,
action: "created",
type: "prompt-version",
user: {
id: "user-123",
name: "Test User",
email: "[email protected]",
},
},
};
await prisma.automationExecution.create({
data: {
id: executionId,
projectId,
triggerId,
automationId,
actionId,
status: ActionExecutionStatus.PENDING,
sourceId: executionId,
input: slackInput,
},
});
await executeWebhook(slackInput);
// Verify custom template was used
expect(mockSlackService.sendMessage).toHaveBeenCalledWith({
client: expect.any(Object),
channelId: "C123456",
blocks: customTemplate,
text: "Langfuse Notification",
});
});
it("should fallback to default message on template error", async () => {
const { SlackService } = await import("@langfuse/shared/src/server");
// Update action with invalid JSON template
await prisma.action.update({
where: { id: actionId },
data: {
config: {
type: "SLACK",
channelId: "C123456",
channelName: "general",
messageTemplate: "invalid-json-template",
} as SlackActionConfig,
},
});
const fullPrompt = await prisma.prompt.findUnique({
where: { id: promptId },
});
const slackInput: WebhookInput = {
projectId,
automationId,
executionId,
payload: {
prompt: fullPrompt as PromptDomain,
action: "created",
type: "prompt-version",
user: {
id: "user-123",
name: "Test User",
email: "[email protected]",
},
},
};
await prisma.automationExecution.create({
data: {
id: executionId,
projectId,
triggerId,
automationId,
actionId,
status: ActionExecutionStatus.PENDING,
sourceId: executionId,
input: slackInput,
},
});
await executeWebhook(slackInput);
// Verify default message was used (should have multiple blocks from SlackMessageBuilder)
expect(mockSlackService.sendMessage).toHaveBeenCalledWith({
client: expect.any(Object),
channelId: "C123456",
blocks: expect.any(Array),
text: "Langfuse Notification",
});
// Verify execution completed successfully despite template error
const execution = await prisma.automationExecution.findUnique({
where: { id: executionId },
});
expect(execution?.status).toBe(ActionExecutionStatus.COMPLETED);
});
it("should disable trigger after 4 consecutive failures", async () => {
const { SlackService } = await import("@langfuse/shared/src/server");
// Mock SlackService to throw errors
mockSlackService.sendMessage.mockRejectedValue(
new Error("Slack API error"),
);
const fullPrompt = await prisma.prompt.findUnique({
where: { id: promptId },
});
// Execute Slack action 5 times to trigger consecutive failures
for (let i = 0; i < 5; i++) {
const executionId = v4();
await prisma.automationExecution.create({
data: {
id: executionId,
projectId,
triggerId,
automationId,
actionId,
status: ActionExecutionStatus.PENDING,
sourceId: v4(),
input: {
promptName: "test-prompt",
promptVersion: 1,
action: "created",
type: "prompt-version",
},
},
});
const slackInput: WebhookInput = {
projectId,
automationId,
executionId,
payload: {
prompt: fullPrompt as PromptDomain,
action: "created",
type: "prompt-version",
user: {
id: "user-123",
name: "Test User",
email: "[email protected]",
},
},
};
await executeWebhook(slackInput);
// Verify execution was marked as error
const execution = await prisma.automationExecution.findUnique({
where: { id: executionId },
});
expect(execution?.status).toBe(ActionExecutionStatus.ERROR);
expect(execution?.error).toContain("Slack API error");
}
// Verify trigger was disabled after 4 consecutive failures
const trigger = await prisma.trigger.findUnique({
where: { id: triggerId },
});
expect(trigger?.status).toBe(JobConfigState.INACTIVE);
});
it("project-notification Slack: records ERROR rows on failure but never disables the trigger", async () => {
mockSlackService.sendMessage.mockRejectedValue(
new Error("Slack API error"),
);
const buildInput = (): WebhookInput => {
const executionId = v4();
return {
projectId,
automationId,
executionId,
payload: {
id: executionId,
timestamp: new Date(),
type: "project-notification",
apiVersion: "v1",
event: {
eventType: "blob-export-failed",
severity: "ALERT",
projectId,
projectName: "Test Project",
resourceId: projectId,
resourceName: "test-bucket",
message: "Blob storage export failed.",
},
},
};
};
// Fail many times over — well past any historical threshold.
for (let i = 0; i < 8; i++) {
const input = buildInput();
await prisma.automationExecution.create({
data: {
id: input.executionId,
projectId,
triggerId,
automationId,
actionId,
status: ActionExecutionStatus.PENDING,
sourceId: projectId,
input: {},
},
});
await executeWebhook(input, { skipValidation: true });
// Each failure is recorded as an ERROR row for visibility.
const execution = await prisma.automationExecution.findUnique({
where: { id: input.executionId },
});
expect(execution?.status).toBe(ActionExecutionStatus.ERROR);
}
// Project-notification channels never auto-disable.
const trigger = await prisma.trigger.findUnique({
where: { id: triggerId },
});
expect(trigger?.status).toBe(JobConfigState.ACTIVE);
});
it("monitor-alert with a JSON-round-tripped (string) timestamp builds the full message, not the fallback", async () => {
// Reconfigure the action as a monitor-alert SLACK action.
await prisma.action.update({
where: { id: actionId },
data: {
config: {
type: "SLACK",
channelId: "C123456",
channelName: "general",
} as SlackActionConfig,
},
});
const monitorInput: WebhookInput = {
projectId,
automationId,
executionId,
payload: {
id: executionId,
timestamp: new Date("2026-05-18T12:01:00.000Z"),
type: "monitor-alert",
apiVersion: "v1",
payload: {
monitorId,
projectId,
severity: "ALERT",
permalink:
"https://cloud.langfuse.com/project/proj_01/monitors/mon_01",
message: { title: "High error rate", body: "errors > 100" },
timestamp: new Date("2026-05-18T12:01:00.000Z"),
fromTimestamp: new Date("2026-05-18T11:55:30.000Z"),
toTimestamp: new Date("2026-05-18T12:00:30.000Z"),
view: "observations",
filters: [],
window: "5m",
},
},
};
// Reproduce the BullMQ wire shape: Dates become ISO strings.
const wireInput: WebhookInput = JSON.parse(JSON.stringify(monitorInput));
await executeWebhook(wireInput, { skipValidation: true });
expect(mockSlackService.sendMessage).toHaveBeenCalledTimes(1);
const sent = mockSlackService.sendMessage.mock.calls[0][0];
// Full monitor message: content nests in the colored attachment; the
// linked title section carries the alert title.
expect(sent.blocks).toEqual([]);
const inner = sent.attachments![0].blocks!;
expect(inner[0].type).toBe("section");
expect(inner[0].text.text).toContain("High error rate");
// Not the fallback: fallback emits a single top-level section and no attachment.
const isFallback =
sent.attachments === undefined &&
sent.blocks.length === 1 &&
sent.blocks[0].text?.text?.startsWith("*Langfuse Notification*");
expect(isFallback).toBe(false);
});
it("monitor-alert: attachment fallback is the alert title; no generic notification text", async () => {
await prisma.action.update({
where: { id: actionId },
data: {
config: {
type: "SLACK",
channelId: "C123456",
channelName: "general",
} as SlackActionConfig,
},
});
const monitorInput: WebhookInput = {
projectId,
automationId,
executionId,
payload: {
id: executionId,
timestamp: new Date("2026-05-18T12:01:00.000Z"),
type: "monitor-alert",
apiVersion: "v1",
payload: {
monitorId,
projectId,
severity: "ALERT",
permalink:
"https://cloud.langfuse.com/project/proj_01/monitors/mon_01",
message: { title: "[ALERT] High error rate", body: "errors > 100" },
timestamp: new Date("2026-05-18T12:01:00.000Z"),
fromTimestamp: new Date("2026-05-18T11:55:30.000Z"),
toTimestamp: new Date("2026-05-18T12:00:30.000Z"),
view: "observations",
filters: [],
window: "5m",
},
},
};
const wireInput: WebhookInput = JSON.parse(JSON.stringify(monitorInput));
await executeWebhook(wireInput, { skipValidation: true });
const sent = mockSlackService.sendMessage.mock.calls[0][0];
expect(sent.text).toBeUndefined();
expect(sent.attachments).toEqual(
expect.arrayContaining([
expect.objectContaining({ fallback: "[ALERT] High error rate" }),
]),
);
});
it("monitor-alert: title escapes Slack mention tokens in the monitor name", async () => {
await prisma.action.update({
where: { id: actionId },
data: {
config: {
type: "SLACK",
channelId: "C123456",
channelName: "general",
} as SlackActionConfig,
},
});
const monitorInput: WebhookInput = {
projectId,
automationId,
executionId,
payload: {
id: executionId,
timestamp: new Date("2026-05-18T12:01:00.000Z"),
type: "monitor-alert",
apiVersion: "v1",
payload: {
monitorId,
projectId,
severity: "ALERT",
permalink:
"https://cloud.langfuse.com/project/proj_01/monitors/mon_01",
message: {
title: "[ALERT] <!channel> investigate",
body: "errors > 100",
},
timestamp: new Date("2026-05-18T12:01:00.000Z"),
fromTimestamp: new Date("2026-05-18T11:55:30.000Z"),
toTimestamp: new Date("2026-05-18T12:00:30.000Z"),
view: "observations",
filters: [],
window: "5m",
},
},
};
const wireInput: WebhookInput = JSON.parse(JSON.stringify(monitorInput));
await executeWebhook(wireInput, { skipValidation: true });
const sent = mockSlackService.sendMessage.mock.calls[0][0];
const titleBlock = sent.attachments![0].blocks![0];
expect(titleBlock.text.text).toContain("<!channel>");
expect(titleBlock.text.text).not.toContain("<!channel>");
});
it("should handle SlackService errors gracefully", async () => {
const { SlackService } = await import("@langfuse/shared/src/server");
// Mock SlackService to throw an error
mockSlackService.getWebClientForProject.mockRejectedValue(
new Error("Failed to get Slack client"),
);
const fullPrompt = await prisma.prompt.findUnique({
where: { id: promptId },
});
const slackInput: WebhookInput = {
projectId,
automationId,
executionId,
payload: {
prompt: fullPrompt as PromptDomain,
action: "created",
type: "prompt-version",
user: {
id: "user-123",
name: "Test User",
email: "[email protected]",
},
},
};
await prisma.automationExecution.create({
data: {
id: executionId,
projectId,
triggerId,
automationId,
actionId,
status: ActionExecutionStatus.PENDING,
sourceId: executionId,
input: slackInput,
},
});
await executeWebhook(slackInput);
// Verify execution was marked as error
const execution = await prisma.automationExecution.findUnique({
where: { id: executionId },
});
expect(execution?.status).toBe(ActionExecutionStatus.ERROR);
expect(execution?.error).toContain("Failed to get Slack client");
});
it("getActionById infra failure propagates for BullMQ retry instead of disabling the trigger", async () => {
const server = await import("@langfuse/shared/src/server");
(server.getActionById as any).mockRejectedValueOnce(
new Error("P2024: connection pool timeout"),
);
const fullPrompt = await prisma.prompt.findUnique({
where: { id: promptId },
});
const slackInput: WebhookInput = {
projectId,
automationId,
executionId,
payload: {
prompt: fullPrompt as PromptDomain,
action: "created",
type: "prompt-version",
user: {
id: "user-123",
name: "Test User",
email: "[email protected]",
},
},
};
// Pre-create the execution row so the catch path can fully disable the
// trigger pre-fix; post-fix the error propagates before the catch.
await prisma.automationExecution.create({
data: {
id: executionId,
projectId,
triggerId,
automationId,
actionId,
status: ActionExecutionStatus.PENDING,
sourceId: executionId,
input: slackInput,
},
});
await expect(executeWebhook(slackInput)).rejects.toThrow();
const trigger = await prisma.trigger.findUnique({
where: { id: triggerId },
});
expect(trigger?.status).toBe(JobConfigState.ACTIVE);
});
});
});