🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { v4 as uuidv4 } from "uuid";
import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";
import {
MonitorService,
type SessionContext,
} from "@langfuse/shared/monitors/server";
import {
MonitorNoDataModeSchema,
MonitorSeveritySchema,
MonitorStatusSchema,
MonitorThresholdOperatorSchema,
} from "@langfuse/shared/monitors";
import { prisma } from "@langfuse/shared/src/db";
import { LangfuseNotFoundError } from "@langfuse/shared";
const baseMonitorInput = (projectId: string) => ({
projectId,
view: "observations" as const,
filters: [],
metric: { measure: "count", aggregation: "count" as const },
window: "5m" as const,
thresholdOperator: MonitorThresholdOperatorSchema.enum.GT,
alertThreshold: 100,
warningThreshold: null,
noData: { mode: MonitorNoDataModeSchema.enum.SHOW_NO_DATA },
renotify: { mode: "OFF" as const },
status: MonitorStatusSchema.enum.ACTIVE,
name: "High error rate",
tags: [],
triggerIds: [],
});
describe("MonitorService (integration)", () => {
let projectId: string;
let creator: SessionContext;
let editor: SessionContext;
beforeAll(async () => {
const org = await createOrgProjectAndApiKey();
projectId = org.projectId;
const creatorUser = await prisma.user.create({
data: {
id: uuidv4(),
email: `monitor-creator-${uuidv4().substring(0, 8)}@test.com`,
name: "Creator",
},
});
const editorUser = await prisma.user.create({
data: {
id: uuidv4(),
email: `monitor-editor-${uuidv4().substring(0, 8)}@test.com`,
name: "Editor",
},
});
creator = { userId: creatorUser.id };
editor = { userId: editorUser.id };
});
afterEach(async () => {
await prisma.monitor.deleteMany({ where: { projectId } });
});
describe("create", () => {
it("creates a monitor and persists derived fields", async () => {
const created = await MonitorService.create(
creator,
baseMonitorInput(projectId),
);
const row = await prisma.monitor.findUnique({
where: { id: created.id },
});
expect(row).not.toBeNull();
expect(row!.windowMs).toBe(5n * 60_000n);
// "5m" is a sub-day window → 1-minute cadence.
expect(row!.cadenceMs).toBe(60_000n);
expect(row!.schedulerBatchId).toBeGreaterThan(0n);
expect(row!.nextRunAt).toBeNull();
expect(row!.severity).toBe(MonitorSeveritySchema.enum.UNKNOWN);
expect(row!.createdBy).toBe(creator.userId);
expect(row!.updatedBy).toBe(creator.userId);
});
it("canonicalizes filter order so permutations share the same schedulerBatchId", async () => {
const filterA = {
column: "environment" as const,
operator: "=" as const,
value: "production",
type: "string" as const,
};
const filterB = {
column: "name" as const,
operator: "=" as const,
value: "faq-bot",
type: "string" as const,
};
const first = await MonitorService.create(creator, {
...baseMonitorInput(projectId),
filters: [filterA, filterB],
});
const second = await MonitorService.create(creator, {
...baseMonitorInput(projectId),
filters: [filterB, filterA],
});
const rows = await prisma.monitor.findMany({
where: { id: { in: [first.id, second.id] } },
});
expect(rows).toHaveLength(2);
expect(rows[0].schedulerBatchId).toBe(rows[1].schedulerBatchId);
});
it("yields a different schedulerBatchId for different projects", async () => {
const otherOrg = await createOrgProjectAndApiKey();
const otherProject = otherOrg.projectId;
const here = await MonitorService.create(
creator,
baseMonitorInput(projectId),
);
const there = await MonitorService.create(
creator,
baseMonitorInput(otherProject),
);
const rows = await prisma.monitor.findMany({
where: { id: { in: [here.id, there.id] } },
});
const idHere = rows.find((r) => r.id === here.id)!.schedulerBatchId;
const idThere = rows.find((r) => r.id === there.id)!.schedulerBatchId;
expect(idHere).not.toBe(idThere);
await prisma.monitor.deleteMany({ where: { projectId: otherProject } });
});
});
describe("update", () => {
it("updates user-editable fields and preserves worker-owned lifecycle state", async () => {
const created = await MonitorService.create(
creator,
baseMonitorInput(projectId),
);
// Simulate the worker writing lifecycle state between create and update.
await prisma.monitor.update({
where: { id: created.id },
data: {
severity: MonitorSeveritySchema.enum.ALERT,
severityChangedAt: new Date(),
alertedAt: new Date(),
lastCompletedAt: new Date(),
},
});
const updated = await MonitorService.update(editor, {
...baseMonitorInput(projectId),
id: created.id,
name: "Renamed",
});
expect(updated.name).toBe("Renamed");
expect(updated.severity).toBe(MonitorSeveritySchema.enum.ALERT);
expect(updated.severityChangedAt).not.toBeNull();
expect(updated.alertedAt).not.toBeNull();
expect(updated.lastCompletedAt).not.toBeNull();
const row = await prisma.monitor.findUnique({
where: { id: created.id },
});
expect(row!.createdBy).toBe(creator.userId);
expect(row!.updatedBy).toBe(editor.userId);
});
it("throws LangfuseNotFoundError when the monitor does not exist", async () => {
await expect(
MonitorService.update(editor, {
...baseMonitorInput(projectId),
id: "mon_missing",
}),
).rejects.toBeInstanceOf(LangfuseNotFoundError);
});
it("flips severity to PAUSED when status leaves ACTIVE", async () => {
const created = await MonitorService.create(
creator,
baseMonitorInput(projectId),
);
expect(created.severity).toBe(MonitorSeveritySchema.enum.UNKNOWN);
const updated = await MonitorService.update(editor, {
...baseMonitorInput(projectId),
id: created.id,
status: MonitorStatusSchema.enum.PAUSED,
});
expect(updated.severity).toBe(MonitorSeveritySchema.enum.PAUSED);
expect(updated.severityChangedAt).not.toBeNull();
});
it("flips severity to UNKNOWN when status returns to ACTIVE", async () => {
const created = await MonitorService.create(creator, {
...baseMonitorInput(projectId),
status: MonitorStatusSchema.enum.PAUSED,
});
expect(created.severity).toBe(MonitorSeveritySchema.enum.PAUSED);
const updated = await MonitorService.update(editor, {
...baseMonitorInput(projectId),
id: created.id,
status: MonitorStatusSchema.enum.ACTIVE,
});
expect(updated.severity).toBe(MonitorSeveritySchema.enum.UNKNOWN);
expect(updated.severityChangedAt).not.toBeNull();
});
it("leaves status and worker-owned severity untouched when status is omitted", async () => {
const created = await MonitorService.create(
creator,
baseMonitorInput(projectId),
);
await prisma.monitor.update({
where: { id: created.id },
data: {
severity: MonitorSeveritySchema.enum.ALERT,
severityChangedAt: new Date(),
},
});
const { status, ...withoutStatus } = baseMonitorInput(projectId);
const updated = await MonitorService.update(editor, {
...withoutStatus,
id: created.id,
name: "Renamed",
});
expect(updated.name).toBe("Renamed");
expect(updated.status).toBe(MonitorStatusSchema.enum.ACTIVE);
expect(updated.severity).toBe(MonitorSeveritySchema.enum.ALERT);
});
it("does not resurrect a paused monitor when status is omitted", async () => {
const created = await MonitorService.create(creator, {
...baseMonitorInput(projectId),
status: MonitorStatusSchema.enum.PAUSED,
});
expect(created.severity).toBe(MonitorSeveritySchema.enum.PAUSED);
const { status, ...withoutStatus } = baseMonitorInput(projectId);
const updated = await MonitorService.update(editor, {
...withoutStatus,
id: created.id,
name: "Renamed",
});
expect(updated.status).toBe(MonitorStatusSchema.enum.PAUSED);
expect(updated.severity).toBe(MonitorSeveritySchema.enum.PAUSED);
});
it("preserves worker-owned severity when status does not change", async () => {
const created = await MonitorService.create(
creator,
baseMonitorInput(projectId),
);
await prisma.monitor.update({
where: { id: created.id },
data: {
severity: MonitorSeveritySchema.enum.ALERT,
severityChangedAt: new Date(),
},
});
const updated = await MonitorService.update(editor, {
...baseMonitorInput(projectId),
id: created.id,
name: "Renamed",
});
expect(updated.severity).toBe(MonitorSeveritySchema.enum.ALERT);
});
it("clears worker lifecycle stamps when update changes schedulerBatchId", async () => {
const created = await MonitorService.create(
creator,
baseMonitorInput(projectId),
);
await prisma.monitor.update({
where: { id: created.id },
data: {
lastPublishedAt: new Date("2026-05-27T11:58:30.000Z"),
lastCompletedAt: null,
},
});
const updated = await MonitorService.update(editor, {
...baseMonitorInput(projectId),
id: created.id,
filters: [
{
column: "environment" as const,
operator: "=" as const,
value: "production",
type: "string" as const,
},
],
});
expect(updated.nextRunAt).toBeNull();
expect(updated.lastPublishedAt).toBeNull();
expect(updated.lastCompletedAt).toBeNull();
});
});
describe("getById / list / delete", () => {
it("returns the monitor by id within the project", async () => {
const created = await MonitorService.create(
creator,
baseMonitorInput(projectId),
);
const fetched = await MonitorService.getById(creator, {
projectId,
id: created.id,
});
expect(fetched.id).toBe(created.id);
});
it("throws LangfuseNotFoundError when fetching from a different project", async () => {
const created = await MonitorService.create(
creator,
baseMonitorInput(projectId),
);
await expect(
MonitorService.getById(creator, {
projectId: "other_project",
id: created.id,
}),
).rejects.toBeInstanceOf(LangfuseNotFoundError);
});
it("paginates list results", async () => {
for (let i = 0; i < 3; i++) {
await MonitorService.create(creator, {
...baseMonitorInput(projectId),
name: `M${i}`,
});
}
const page = await MonitorService.list(creator, {
projectId,
orderBy: null,
limit: 2,
page: 1,
});
expect(page.totalCount).toBe(3);
expect(page.monitors).toHaveLength(2);
});
it("deletes a monitor and getById then throws LangfuseNotFoundError", async () => {
const created = await MonitorService.create(
creator,
baseMonitorInput(projectId),
);
await MonitorService.delete(creator, { projectId, id: created.id });
await expect(
MonitorService.getById(creator, { projectId, id: created.id }),
).rejects.toBeInstanceOf(LangfuseNotFoundError);
});
it.each(["name", "status", "severity", "createdAt", "updatedAt"] as const)(
"lists by %s (non-nullable column) without a PrismaClientValidationError",
async (column) => {
// Verifies whether Prisma accepts `{ sort, nulls: "last" }` on
// non-nullable columns. If it doesn't, this throws
// PrismaClientValidationError at runtime and we need to gate `nulls`
// on the nullable subset instead.
await MonitorService.create(creator, baseMonitorInput(projectId));
const result = await MonitorService.list(creator, {
projectId,
orderBy: { column, order: "ASC" },
page: 1,
limit: 10,
});
expect(result.monitors.length).toBeGreaterThan(0);
},
);
it.each(["alertedAt", "severityChangedAt"] as const)(
"sorts %s with NULLS LAST regardless of direction",
async (column) => {
const dbColumn =
column === "alertedAt" ? "alertedAt" : "severityChangedAt";
// Three monitors: one never-touched (NULL), one older, one newer.
const neverTouched = await MonitorService.create(creator, {
...baseMonitorInput(projectId),
name: "never",
});
const older = await MonitorService.create(creator, {
...baseMonitorInput(projectId),
name: "older",
});
const newer = await MonitorService.create(creator, {
...baseMonitorInput(projectId),
name: "newer",
});
await prisma.monitor.update({
where: { id: older.id },
data: { [dbColumn]: new Date("2026-01-01T00:00:00.000Z") },
});
await prisma.monitor.update({
where: { id: newer.id },
data: { [dbColumn]: new Date("2026-06-01T00:00:00.000Z") },
});
const desc = await MonitorService.list(creator, {
projectId,
orderBy: { column, order: "DESC" },
page: 1,
limit: 10,
});
expect(desc.monitors.map((m) => m.id)).toEqual([
newer.id,
older.id,
neverTouched.id,
]);
const asc = await MonitorService.list(creator, {
projectId,
orderBy: { column, order: "ASC" },
page: 1,
limit: 10,
});
expect(asc.monitors.map((m) => m.id)).toEqual([
older.id,
newer.id,
neverTouched.id,
]);
},
);
});
});