🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { describe, expect, it } from "vitest";
import { type FilterState } from "@langfuse/shared";
import {
type Monitor,
MonitorNoDataModeSchema,
MonitorSeveritySchema,
ListMonitorFilterSchema,
MonitorStatusSchema,
MonitorThresholdOperatorSchema,
} from "@langfuse/shared/monitors";
import { getMonitorFilterConfig } from "@/src/features/filters/config/monitors-config";
import { __test } from "./MonitorsTable";
const { filterStateToListMonitorFilter, buildStatusToggleUpdate } = __test;
/** monitorFixture builds a full Monitor row with the given status. */
const monitorFixture = (status: Monitor["status"]): Monitor => ({
id: "mon-1",
projectId: "project-1",
view: "observations",
filters: [],
metric: { measure: "count", aggregation: "count" },
window: "5m",
thresholdOperator: MonitorThresholdOperatorSchema.enum.GT,
alertThreshold: 10,
warningThreshold: null,
noData: { mode: MonitorNoDataModeSchema.enum.SHOW_NO_DATA },
renotify: { mode: "OFF" },
name: "My Monitor",
tags: ["prod"],
triggerIds: ["t-a"],
status,
severity: MonitorSeveritySchema.enum.OK,
severityChangedAt: null,
alertedAt: null,
createdAt: new Date(),
updatedAt: new Date(),
createdBy: null,
updatedBy: null,
nextRunAt: new Date(),
lastPublishedAt: null,
lastClaimedAt: null,
lastCompletedAt: null,
});
describe("getMonitorFilterConfig", () => {
it("shows the evaluator facet only when evaluator-specific alerts exist", () => {
expect(
getMonitorFilterConfig(false).facets.some(
(facet) => facet.column === "evaluatorId",
),
).toBe(false);
expect(
getMonitorFilterConfig(true).facets.some(
(facet) => facet.column === "evaluatorId",
),
).toBe(true);
});
it("includes the evaluator ID in the option hover title", () => {
const evaluatorFacet = getMonitorFilterConfig(true).facets.find(
(facet) => facet.column === "evaluatorId" && facet.type === "categorical",
);
expect(
evaluatorFacet?.type === "categorical"
? evaluatorFacet.getOptionTitle?.("evaluator-1", "Answer quality")
: undefined,
).toBe("Answer quality (evaluator-1)");
});
});
describe("filterStateToListMonitorFilter", () => {
it.each([
{ column: "severity", type: "stringOptions", value: ["ALERT"] },
{ column: "evaluatorId", type: "stringOptions", value: ["eval-1"] },
{ column: "tags", type: "arrayOptions", value: ["prod"] },
] as const)(
"matches nothing for an empty $column without losing other facets",
({ column, type, value }) => {
const other = (
[
{
column: "severity",
type: "stringOptions",
operator: "any of",
value: ["WARNING"],
},
{
column: "evaluatorId",
type: "stringOptions",
operator: "any of",
value: ["eval-2"],
},
{
column: "tags",
type: "arrayOptions",
operator: "all of",
value: ["critical"],
},
] satisfies FilterState
).filter((filter) => filter.column !== column);
const state: FilterState = [
...other,
{ column, type, operator: "any of", value: [...value] },
{ column, type, operator: "none of", value: [...value] },
];
const original = structuredClone(state);
const request = filterStateToListMonitorFilter(state);
expect(ListMonitorFilterSchema.safeParse(request).success).toBe(true);
expect(request).toEqual([
...other.filter((filter) => filter.column !== "severity"),
{
column: "severity",
type: "stringOptions",
operator: "none of",
value: MonitorSeveritySchema.options,
},
]);
expect(state).toEqual(original);
},
);
it("passes severity `any of` through with values intact", () => {
const state: FilterState = [
{
type: "stringOptions",
column: "severity",
operator: "any of",
value: [
MonitorSeveritySchema.enum.ALERT,
MonitorSeveritySchema.enum.WARNING,
],
},
];
expect(filterStateToListMonitorFilter(state)).toEqual([
{
type: "stringOptions",
column: "severity",
operator: "any of",
value: [
MonitorSeveritySchema.enum.ALERT,
MonitorSeveritySchema.enum.WARNING,
],
},
]);
});
it("expands NO_DATA to (NO_DATA, UNKNOWN) on the severity column", () => {
const state: FilterState = [
{
type: "stringOptions",
column: "severity",
operator: "any of",
value: [MonitorSeveritySchema.enum.NO_DATA],
},
];
expect(filterStateToListMonitorFilter(state)).toEqual([
{
type: "stringOptions",
column: "severity",
operator: "any of",
value: [
MonitorSeveritySchema.enum.NO_DATA,
MonitorSeveritySchema.enum.UNKNOWN,
],
},
]);
});
it("expands NO_DATA on `none of` too", () => {
const state: FilterState = [
{
type: "stringOptions",
column: "severity",
operator: "none of",
value: [MonitorSeveritySchema.enum.NO_DATA],
},
];
expect(filterStateToListMonitorFilter(state)).toEqual([
{
type: "stringOptions",
column: "severity",
operator: "none of",
value: [
MonitorSeveritySchema.enum.NO_DATA,
MonitorSeveritySchema.enum.UNKNOWN,
],
},
]);
});
it("does not duplicate UNKNOWN when both NO_DATA and UNKNOWN are already present", () => {
const state: FilterState = [
{
type: "stringOptions",
column: "severity",
operator: "any of",
value: [
MonitorSeveritySchema.enum.NO_DATA,
MonitorSeveritySchema.enum.UNKNOWN,
MonitorSeveritySchema.enum.ALERT,
],
},
];
expect(filterStateToListMonitorFilter(state)).toEqual([
{
type: "stringOptions",
column: "severity",
operator: "any of",
value: [
MonitorSeveritySchema.enum.NO_DATA,
MonitorSeveritySchema.enum.UNKNOWN,
MonitorSeveritySchema.enum.ALERT,
],
},
]);
});
it("passes evaluator ID rows through unchanged", () => {
const state: FilterState = [
{
type: "stringOptions",
column: "evaluatorId",
operator: "any of",
value: ["eval-1"],
},
];
expect(filterStateToListMonitorFilter(state)).toEqual(state);
});
it("passes tags rows through unchanged", () => {
const state: FilterState = [
{
type: "arrayOptions",
column: "tags",
operator: "any of",
value: ["prod"],
},
];
expect(filterStateToListMonitorFilter(state)).toEqual(state);
});
it("collapses to no filter when a row has an unrecognized column", () => {
const state: FilterState = [
{
type: "stringOptions",
column: "bogus",
operator: "any of",
value: ["x"],
},
{
type: "arrayOptions",
column: "tags",
operator: "any of",
value: ["prod"],
},
];
expect(filterStateToListMonitorFilter(state)).toEqual([]);
});
it("collapses to no filter when a column is duplicated", () => {
const state: FilterState = [
{
type: "stringOptions",
column: "severity",
operator: "any of",
value: [MonitorSeveritySchema.enum.ALERT],
},
{
type: "stringOptions",
column: "severity",
operator: "none of",
value: [MonitorSeveritySchema.enum.PAUSED],
},
];
expect(filterStateToListMonitorFilter(state)).toEqual([]);
});
});
describe("buildStatusToggleUpdate", () => {
it("ACTIVE monitor: flips status to PAUSED", () => {
expect(
buildStatusToggleUpdate(monitorFixture(MonitorStatusSchema.enum.ACTIVE))
.status,
).toBe(MonitorStatusSchema.enum.PAUSED);
});
it("PAUSED monitor: flips status to ACTIVE", () => {
expect(
buildStatusToggleUpdate(monitorFixture(MonitorStatusSchema.enum.PAUSED))
.status,
).toBe(MonitorStatusSchema.enum.ACTIVE);
});
it("carries the full config so only status changes", () => {
const monitor = monitorFixture(MonitorStatusSchema.enum.ACTIVE);
expect(buildStatusToggleUpdate(monitor)).toEqual({
id: monitor.id,
projectId: monitor.projectId,
view: monitor.view,
filters: monitor.filters,
metric: monitor.metric,
window: monitor.window,
thresholdOperator: monitor.thresholdOperator,
alertThreshold: monitor.alertThreshold,
warningThreshold: monitor.warningThreshold,
noData: monitor.noData,
renotify: monitor.renotify,
name: monitor.name,
tags: monitor.tags,
triggerIds: monitor.triggerIds,
status: MonitorStatusSchema.enum.PAUSED,
});
});
});