Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(evals): add evaluator alert management (#16903)

Tobias Wochingeradd518fe2880
diff --git a/packages/shared/src/features/monitors/server.ts b/packages/shared/src/features/monitors/server.ts
index 0027328e71..69f1e1f399 100644
--- a/packages/shared/src/features/monitors/server.ts
+++ b/packages/shared/src/features/monitors/server.ts
@@ -3,6 +3,7 @@
* scheduler DTOs. */
export * from "./index";
export * from "./service";
+export * from "./service/evaluatorFilterMatching";
export * from "./scheduler/types";
export * from "./scheduler/scheduler";
export * from "./processor/processor";
diff --git a/packages/shared/src/features/monitors/service/evaluatorFilterMatching.ts b/packages/shared/src/features/monitors/service/evaluatorFilterMatching.ts
new file mode 100644
index 0000000000..945446cc0b
--- /dev/null
+++ b/packages/shared/src/features/monitors/service/evaluatorFilterMatching.ts
@@ -0,0 +1,64 @@
+import { Prisma, type PrismaClient } from "../../../db";
+
+/**
+ * Matches an evaluator ID against the positive evaluator filter operators
+ * that can keep a monitor linked to that evaluator.
+ */
+export const evaluatorFilterMatchesSql = (
+ filterRow: Prisma.Sql,
+ evaluatorId: Prisma.Sql,
+): Prisma.Sql => Prisma.sql`
+ ${filterRow}->>'column' = 'evaluatorId'
+ AND (
+ (
+ ${filterRow}->>'type' = 'string'
+ AND (
+ (${filterRow}->>'operator' = '=' AND ${evaluatorId} = ${filterRow}->>'value')
+ OR (${filterRow}->>'operator' = 'contains' AND strpos(${evaluatorId}, ${filterRow}->>'value') > 0)
+ OR (${filterRow}->>'operator' = 'starts with' AND starts_with(${evaluatorId}, ${filterRow}->>'value'))
+ )
+ )
+ OR (
+ ${filterRow}->>'type' = 'stringOptions'
+ AND ${filterRow}->>'operator' = 'any of'
+ AND ${filterRow}->'value' @> jsonb_build_array(${evaluatorId})
+ )
+ )
+`;
+
+/** Returns monitor IDs whose evaluator filter matches any supplied evaluator. */
+export const findMonitorIdsLinkedToEvaluators = async (
+ client: Pick<PrismaClient, "$queryRaw">,
+ params: {
+ projectId: string;
+ evaluatorIds: string[];
+ limit?: number;
+ },
+): Promise<string[]> => {
+ if (params.evaluatorIds.length === 0) return [];
+
+ const selectedEvaluators = Prisma.join(
+ params.evaluatorIds.map((id) => Prisma.sql`(${id})`),
+ );
+ const matchesEvaluator = evaluatorFilterMatchesSql(
+ Prisma.raw("filter_row"),
+ Prisma.raw("selected_evaluator.id"),
+ );
+ const limit = params.limit ? Prisma.sql`LIMIT ${params.limit}` : Prisma.empty;
+
+ const rows = await client.$queryRaw<{ id: string }[]>(Prisma.sql`
+ SELECT monitor.id
+ FROM monitors AS monitor
+ WHERE monitor.project_id = ${params.projectId}
+ AND EXISTS (
+ SELECT 1
+ FROM jsonb_array_elements(monitor.filters) AS filter_row
+ CROSS JOIN (VALUES ${selectedEvaluators}) AS selected_evaluator(id)
+ WHERE ${matchesEvaluator}
+ )
+ ORDER BY monitor.updated_at DESC
+ ${limit}
+ `);
+
+ return rows.map(({ id }) => id);
+};
diff --git a/packages/shared/src/features/monitors/service/helpers.test.ts b/packages/shared/src/features/monitors/service/helpers.test.ts
index ab08549908..ff1bf57591 100644
--- a/packages/shared/src/features/monitors/service/helpers.test.ts
+++ b/packages/shared/src/features/monitors/service/helpers.test.ts
@@ -184,6 +184,48 @@ describe("toPrismaWhere", () => {
});
});
+ it("translates evaluator IDs into monitor-filter JSON predicates", () => {
+ const filter: ListMonitorFilter = [
+ {
+ type: "stringOptions",
+ column: "evaluatorId",
+ operator: "any of",
+ value: ["eval-1", "eval-2"],
+ },
+ ];
+
+ expect(
+ toPrismaWhere("proj_01", filter, ["monitor-1", "monitor-2"]),
+ ).toEqual({
+ projectId: "proj_01",
+ AND: [
+ {
+ id: { in: ["monitor-1", "monitor-2"] },
+ },
+ ],
+ });
+ });
+
+ it("negates excluded evaluator IDs", () => {
+ const filter: ListMonitorFilter = [
+ {
+ type: "stringOptions",
+ column: "evaluatorId",
+ operator: "none of",
+ value: ["eval-1"],
+ },
+ ];
+
+ expect(toPrismaWhere("proj_01", filter, ["monitor-1"])).toEqual({
+ projectId: "proj_01",
+ AND: [
+ {
+ id: { notIn: ["monitor-1"] },
+ },
+ ],
+ });
+ });
+
it("translates tags operators to Prisma array predicates", () => {
const filter: ListMonitorFilter = [
{
diff --git a/packages/shared/src/features/monitors/service/helpers.ts b/packages/shared/src/features/monitors/service/helpers.ts
index 6d0b808d62..25e5ae9bef 100644
--- a/packages/shared/src/features/monitors/service/helpers.ts
+++ b/packages/shared/src/features/monitors/service/helpers.ts
@@ -55,9 +55,18 @@ export const toPrismaOrderBy = (
export const toPrismaWhere = (
projectId: string,
filter: ListMonitorFilter | undefined,
+ evaluatorMonitorIds?: string[],
): Prisma.MonitorWhereInput => {
const and: Prisma.MonitorWhereInput[] = [];
for (const f of filter ?? []) {
+ if (f.column === "evaluatorId") {
+ and.push(
+ f.operator === "any of"
+ ? { id: { in: evaluatorMonitorIds ?? [] } }
+ : { id: { notIn: evaluatorMonitorIds ?? [] } },
+ );
+ continue;
+ }
if (f.type === "stringOptions") {
and.push(
f.operator === "any of"
diff --git a/packages/shared/src/features/monitors/service/service.ts b/packages/shared/src/features/monitors/service/service.ts
index e64a76b7cd..555198f53f 100644
--- a/packages/shared/src/features/monitors/service/service.ts
+++ b/packages/shared/src/features/monitors/service/service.ts
@@ -28,6 +28,10 @@ import {
type SessionContext,
type UpdateMonitor,
} from "./types";
+import {
+ evaluatorFilterMatchesSql,
+ findMonitorIdsLinkedToEvaluators,
+} from "./evaluatorFilterMatching";
/**
* MonitorService manages all of the Monitors that produce MonitorAlerts on
@@ -161,7 +165,20 @@ export class MonitorService {
): Promise<{ monitors: Monitor[]; totalCount: number }> {
const skip =
input.page && input.limit ? (input.page - 1) * input.limit : undefined;
- const where = toPrismaWhere(input.projectId, input.filter);
+ const evaluatorFilter = input.filter?.find(
+ (filter) => filter.column === "evaluatorId",
+ );
+ const evaluatorMonitorIds = evaluatorFilter
+ ? await findMonitorIdsLinkedToEvaluators(prisma, {
+ projectId: input.projectId,
+ evaluatorIds: evaluatorFilter.value,
+ })
+ : undefined;
+ const where = toPrismaWhere(
+ input.projectId,
+ input.filter,
+ evaluatorMonitorIds,
+ );
const [monitors, totalCount] = await Promise.all([
prisma.monitor.findMany({
@@ -179,15 +196,36 @@ export class MonitorService {
public static async getFilterOptions(
_session: SessionContext,
input: GetMonitorFilterOptions,
- ): Promise<{ tags: { value: string }[] }> {
- const rows = await prisma.$queryRaw<{ value: string }[]>`
- SELECT tags.tag AS value
- FROM monitors, UNNEST(monitors.tags) AS tags(tag)
- WHERE monitors.project_id = ${input.projectId}
- GROUP BY tags.tag
- ORDER BY tags.tag ASC;
- `;
- return { tags: rows };
+ ): Promise<{
+ tags: { value: string }[];
+ evaluators: { value: string; displayValue: string }[];
+ }> {
+ const evaluatorFilterMatch = evaluatorFilterMatchesSql(
+ Prisma.raw("filter_row"),
+ Prisma.raw("e.id"),
+ );
+ const [tags, evaluators] = await Promise.all([
+ prisma.$queryRaw<{ value: string }[]>`
+ SELECT tags.tag AS value
+ FROM monitors, UNNEST(monitors.tags) AS tags(tag)
+ WHERE monitors.project_id = ${input.projectId}
+ GROUP BY tags.tag
+ ORDER BY tags.tag ASC;
+ `,
+ prisma.$queryRaw<{ value: string; displayValue: string }[]>`
+ SELECT DISTINCT
+ e.id AS value,
+ e.name AS "displayValue"
+ FROM monitors AS m
+ CROSS JOIN LATERAL jsonb_array_elements(m.filters) AS filter_row
+ JOIN evaluators AS e
+ ON e.project_id = m.project_id
+ AND ${evaluatorFilterMatch}
+ WHERE m.project_id = ${input.projectId}
+ ORDER BY "displayValue" ASC, value ASC;
+ `,
+ ]);
+ return { tags, evaluators };
}
public static async delete(
diff --git a/packages/shared/src/features/monitors/service/types.ts b/packages/shared/src/features/monitors/service/types.ts
index c59e0dbfd9..f58e4d26a4 100644
--- a/packages/shared/src/features/monitors/service/types.ts
+++ b/packages/shared/src/features/monitors/service/types.ts
@@ -106,6 +106,17 @@ export type ListMonitorSeverityFilter = z.infer<
typeof ListMonitorSeverityFilterSchema
>;
+/** ListMonitorEvaluatorIdFilterSchema matches alerts connected to selected evaluators. */
+export const ListMonitorEvaluatorIdFilterSchema = z.object({
+ type: z.literal("stringOptions"),
+ column: z.literal("evaluatorId"),
+ operator: z.enum(["any of", "none of"]),
+ value: z.array(z.string()).min(1),
+});
+export type ListMonitorEvaluatorIdFilter = z.infer<
+ typeof ListMonitorEvaluatorIdFilterSchema
+>;
+
/** ListMonitorTagsFilterSchema is the tags row of a ListMonitorFilter. */
export const ListMonitorTagsFilterSchema = z.object({
type: z.literal("arrayOptions"),
@@ -124,6 +135,7 @@ export const ListMonitorFilterSchema = z
.array(
z.discriminatedUnion("column", [
ListMonitorSeverityFilterSchema,
+ ListMonitorEvaluatorIdFilterSchema,
ListMonitorTagsFilterSchema,
]),
)
diff --git a/packages/shared/src/features/query/dataModel.ts b/packages/shared/src/features/query/dataModel.ts
index 2a20ff147d..1d3828f4ac 100644
--- a/packages/shared/src/features/query/dataModel.ts
+++ b/packages/shared/src/features/query/dataModel.ts
@@ -13,6 +13,7 @@ import {
eventsTableTraceNameSqlForAlias,
} from "../../eventsTable";
import { LISTABLE_SCORE_TYPES } from "../../domain/scores";
+import { EvalExecutionMetadataKey } from "../evals/evalExecutionMetadata";
// The data model defines all available dimensions, measures, and the timeDimension for a given view.
// Make sure to update web/src/features/dashboard/lib/dashboardUiTableToViewMapping.ts if you make changes
@@ -369,6 +370,27 @@ export const eventsTracesView: ViewDeclarationType = {
baseCte: `events_core events_traces`,
};
+const createEvaluatorDimensions = (
+ tableAlias: string,
+): DimensionsDeclarationType => ({
+ evaluatorId: {
+ sql: `coalesce(nullIf(${tableAlias}.evaluator_id, ''), ${tableAlias}.evaluation_rule_id)`,
+ alias: "evaluatorId",
+ type: "string",
+ description:
+ "Identifier of the evaluator, falling back to its legacy evaluation rule identifier.",
+ highCardinality: true,
+ uiHidden: true,
+ },
+ isEvaluatorTest: {
+ sql: `${tableAlias}.evaluator_execution_is_test`,
+ alias: "isEvaluatorTest",
+ type: "boolean",
+ description: "Whether this row belongs to an evaluator test run.",
+ uiHidden: true,
+ },
+});
+
export const observationsView: ViewDeclarationType = {
name: "observations",
description:
@@ -903,6 +925,13 @@ const createScoreSpecificDimensions = (
highCardinality: true,
uiHidden: true,
},
+ isEvaluatorTest: {
+ sql: `toBool(${tableAlias}.metadata['${EvalExecutionMetadataKey.EVALUATOR_TEST}'] = 'true')`,
+ alias: "isEvaluatorTest",
+ type: "boolean",
+ description: "Whether this score belongs to an evaluator test run.",
+ uiHidden: true,
+ },
source: {
sql: `${tableAlias}.source`,
alias: "source",
@@ -1224,6 +1253,7 @@ export const eventsObservationsView: ViewDeclarationType = {
highCardinality: true,
uiHidden: true,
},
+ ...createEvaluatorDimensions("events_observations"),
traceId: {
sql: "events_observations.trace_id",
alias: "traceId",
diff --git a/packages/shared/src/features/query/server/queryBuilder.filterValidation.test.ts b/packages/shared/src/features/query/server/queryBuilder.filterValidation.test.ts
index afadf42dfd..5ff42e84bf 100644
--- a/packages/shared/src/features/query/server/queryBuilder.filterValidation.test.ts
+++ b/packages/shared/src/features/query/server/queryBuilder.filterValidation.test.ts
@@ -314,6 +314,57 @@ describe("queryBuilder filter type validation", () => {
},
);
+ it.each(["v1", "v2"] as const)(
+ "lowers evaluator test filters in the %s scores view",
+ async (version) => {
+ const { query } = await buildQueryWithFilter(
+ {
+ column: "isEvaluatorTest",
+ operator: "=",
+ value: false,
+ type: "boolean",
+ },
+ { view: "scores-numeric" },
+ version,
+ );
+
+ expect(query).toContain("scores_numeric.metadata['evaluator_test']");
+ expect(query).toContain(": Boolean}");
+ },
+ );
+
+ it("lowers evaluator identity and test filters in the v2 observations view", async () => {
+ const evaluatorIdQuery = await buildQueryWithFilter(
+ {
+ column: "evaluatorId",
+ operator: "is not empty",
+ value: "",
+ type: "string",
+ },
+ { view: "observations" },
+ "v2",
+ );
+ const evaluatorTestQuery = await buildQueryWithFilter(
+ {
+ column: "isEvaluatorTest",
+ operator: "=",
+ value: false,
+ type: "boolean",
+ },
+ { view: "observations" },
+ "v2",
+ );
+
+ expect(evaluatorIdQuery.query).toContain(
+ "coalesce(nullIf(events_observations.evaluator_id, ''), events_observations.evaluation_rule_id)",
+ );
+ expect(evaluatorTestQuery.query).toContain(
+ "events_observations.evaluator_execution_is_test",
+ );
+ expect(evaluatorIdQuery.query).toContain("!= ''");
+ expect(evaluatorTestQuery.query).toContain(": Boolean}");
+ });
+
it("lowers the semantic-root observation filter only in the v2 events view", async () => {
const { query } = await buildQueryWithFilter(
{
diff --git a/packages/shared/src/interfaces/filters.ts b/packages/shared/src/interfaces/filters.ts
index d84ab3743c..ac59691a60 100644
--- a/packages/shared/src/interfaces/filters.ts
+++ b/packages/shared/src/interfaces/filters.ts
@@ -3,7 +3,14 @@ import { z } from "zod";
// Make sure to update the InMemoryFilterService if you add new filter types
export const filterOperators = {
datetime: [">", "<", ">=", "<="],
- string: ["=", "contains", "does not contain", "starts with", "ends with"],
+ string: [
+ "=",
+ "contains",
+ "does not contain",
+ "starts with",
+ "ends with",
+ "is not empty",
+ ],
stringOptions: ["any of", "none of"],
categoryOptions: ["any of", "none of"],
arrayOptions: ["any of", "none of", "all of"],
@@ -71,7 +78,7 @@ export const stringObjectFilter = z.object({
type: z.literal("stringObject"),
column: z.string(),
key: z.string(), // eg metadata --> "environment"
- operator: z.enum(filterOperators.string),
+ operator: z.enum(filterOperators.stringObject),
value: z.string(),
});
export const numberObjectFilter = z.object({
diff --git a/packages/shared/src/server/filterToPrisma.ts b/packages/shared/src/server/filterToPrisma.ts
index 574a154653..4a315bdc08 100644
--- a/packages/shared/src/server/filterToPrisma.ts
+++ b/packages/shared/src/server/filterToPrisma.ts
@@ -124,6 +124,10 @@ export function tableColumnsToSqlFilter(
logger.warn("Position-in-trace filters are not supported in postgres");
throw new Error("Position-in-trace filters not supported in postgres");
}
+ if (filter.type === "string" && filter.operator === "is not empty") {
+ return Prisma.sql`(${filterAndColumn.internalColumn} IS NOT NULL AND ${filterAndColumn.internalColumn} <> '')`;
+ }
+
const jsonKeyPrisma =
filter.type === "stringObject" || filter.type === "numberObject"
? Prisma.sql`->>${filter.key}`
diff --git a/packages/shared/src/server/prismaFilter.ts b/packages/shared/src/server/prismaFilter.ts
index 7441406d72..89dbdb30d6 100644
--- a/packages/shared/src/server/prismaFilter.ts
+++ b/packages/shared/src/server/prismaFilter.ts
@@ -46,6 +46,8 @@ export function stringFilterToPrisma(
return { startsWith: filter.value, ...insensitive };
case "ends with":
return { endsWith: filter.value, ...insensitive };
+ case "is not empty":
+ return { not: "" };
}
}
diff --git a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts
index 0842311b47..71d8aded1a 100644
--- a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts
+++ b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts
@@ -107,6 +107,9 @@ export class StringFilter implements Filter {
case "ends with":
query = `endsWith(${fieldWithPrefix}, {${varName}: String})`;
break;
+ case "is not empty":
+ query = `(${fieldWithPrefix} != '' AND ${fieldWithPrefix} IS NOT NULL)`;
+ break;
case FTS_MATCH_OPERATOR:
assertValidFtsMatchFilter({
filterType: "string",
@@ -132,8 +135,8 @@ export class StringFilter implements Filter {
}
return {
- query: query,
- params: { [varName]: this.value },
+ query,
+ params: this.operator === "is not empty" ? {} : { [varName]: this.value },
};
}
}
diff --git a/packages/shared/src/server/services/InMemoryFilterService.ts b/packages/shared/src/server/services/InMemoryFilterService.ts
index cc9fdc892b..53463e7b43 100644
--- a/packages/shared/src/server/services/InMemoryFilterService.ts
+++ b/packages/shared/src/server/services/InMemoryFilterService.ts
@@ -143,6 +143,8 @@ export class InMemoryFilterService {
return strValue.startsWith(filterValue);
case "ends with":
return strValue.endsWith(filterValue);
+ case "is not empty":
+ return strValue.length > 0;
default:
logger.error("Unsupported string filter operator", {
operator,
diff --git a/packages/shared/src/tableDefinitions/monitorsTable.ts b/packages/shared/src/tableDefinitions/monitorsTable.ts
index 936c0a0c68..5e4082363b 100644
--- a/packages/shared/src/tableDefinitions/monitorsTable.ts
+++ b/packages/shared/src/tableDefinitions/monitorsTable.ts
@@ -34,4 +34,11 @@ export const monitorsTableCols: ColumnDefinition[] = [
internal: 'm."tags"',
options: [],
},
+ {
+ name: "Evaluator ID",
+ id: "evaluatorId",
+ type: "stringOptions",
+ internal: "evaluatorId",
+ options: [],
+ },
];
diff --git a/packages/shared/src/tableDefinitions/types.ts b/packages/shared/src/tableDefinitions/types.ts
index 0a3a61e95e..8203c4e496 100644
--- a/packages/shared/src/tableDefinitions/types.ts
+++ b/packages/shared/src/tableDefinitions/types.ts
@@ -64,6 +64,8 @@ export type ColumnDefinition =
step?: number;
/** Minimum value for number inputs. */
min?: number;
+ /** Optional suggestions whose values are persisted while displayValue is rendered. */
+ options?: Array<SingleValueOption>;
}
| {
name: string;
diff --git a/web/src/__tests__/server/monitors.servertest.ts b/web/src/__tests__/server/monitors.servertest.ts
index 84293b9121..818fc03958 100644
--- a/web/src/__tests__/server/monitors.servertest.ts
+++ b/web/src/__tests__/server/monitors.servertest.ts
@@ -167,6 +167,118 @@ describe("monitors trpc", () => {
expect(list.monitors.map((m) => m.id)).toContain(created.id);
});
+ it("does not suggest a title when AI features are unavailable", async () => {
+ const { project, caller } = await prepare();
+
+ await expect(
+ caller.monitors.suggestName({
+ projectId: project.id,
+ description: "Count of observations is above 100",
+ }),
+ ).resolves.toBeNull();
+ });
+
+ it.each([
+ {
+ name: "equals",
+ filter: {
+ column: "evaluatorId",
+ type: "string" as const,
+ operator: "=" as const,
+ value: "evaluator-1",
+ },
+ },
+ {
+ name: "contains",
+ filter: {
+ column: "evaluatorId",
+ type: "string" as const,
+ operator: "contains" as const,
+ value: "valuator-1",
+ },
+ },
+ {
+ name: "starts with",
+ filter: {
+ column: "evaluatorId",
+ type: "string" as const,
+ operator: "starts with" as const,
+ value: "evaluator-",
+ },
+ },
+ {
+ name: "any of",
+ filter: {
+ column: "evaluatorId",
+ type: "stringOptions" as const,
+ operator: "any of" as const,
+ value: ["evaluator-2", "evaluator-1"],
+ },
+ },
+ ])(
+ "returns alerts whose $name filter matches the evaluator id",
+ async ({ name, filter }) => {
+ const { project, caller } = await prepare();
+ const monitor = await caller.monitors.create({
+ ...validMonitorInput(project.id),
+ name: `${name} evaluator`,
+ filters: [filter],
+ });
+
+ const linked = await caller.monitors.linkedEvaluatorAlerts({
+ projectId: project.id,
+ evaluatorId: "evaluator-1",
+ });
+
+ expect(linked.data.map(({ id }) => id)).toEqual([monitor.id]);
+ expect(linked.hasMore).toBe(false);
+ },
+ );
+
+ it("returns aggregate evaluator spend alerts", async () => {
+ const { project, caller } = await prepare();
+
+ const aggregate = await caller.monitors.create({
+ ...validMonitorInput(project.id),
+ name: "All evaluator spend",
+ metric: { measure: "totalCost", aggregation: "sum" },
+ filters: [
+ {
+ column: "evaluatorId",
+ type: "string",
+ operator: "is not empty",
+ value: "",
+ },
+ {
+ column: "isEvaluatorTest",
+ type: "boolean",
+ operator: "=",
+ value: false,
+ },
+ ],
+ });
+
+ await expect(
+ caller.monitors.linkedAllEvaluatorSpendAlerts({
+ projectId: project.id,
+ }),
+ ).resolves.toEqual({
+ data: [
+ {
+ id: aggregate.id,
+ name: "All evaluator spend",
+ status: MonitorStatusSchema.enum.ACTIVE,
+ severity: MonitorSeveritySchema.enum.UNKNOWN,
+ metric: { measure: "totalCost", aggregation: "sum" },
+ thresholdOperator: MonitorThresholdOperatorSchema.enum.GT,
+ alertThreshold: 100,
+ alertedAt: null,
+ },
+ ],
+ hasMore: false,
+ });
+ });
+
it("returns ERROR_BAD_QUERY monitors verbatim (scheduler-owned status survives a read)", async () => {
const { project, caller } = await prepare();
@@ -207,7 +319,7 @@ describe("monitors trpc", () => {
expect(created.id).toBeDefined();
});
- it("allows monitors.all from VIEWER role (read-only scope)", async () => {
+ it("allows monitor reads from VIEWER role (read-only scope)", async () => {
const { project, caller } = await prepare({ projectRole: "VIEWER" });
const list = await caller.monitors.all({
@@ -216,7 +328,16 @@ describe("monitors trpc", () => {
page: 1,
limit: 50,
});
+ const linkedAlerts = await caller.monitors.linkedEvaluatorAlerts({
+ projectId: project.id,
+ evaluatorId: "evaluator-1",
+ });
+ const spendAlerts = await caller.monitors.linkedAllEvaluatorSpendAlerts({
+ projectId: project.id,
+ });
expect(list.totalCount).toBe(0);
+ expect(linkedAlerts).toEqual({ data: [], hasMore: false });
+ expect(spendAlerts).toEqual({ data: [], hasMore: false });
});
});
@@ -415,12 +536,38 @@ describe("monitors trpc", () => {
expect(result.monitors.map((m) => m.name)).toEqual(["Tagged"]);
});
- it("filterOptions returns distinct tags for the project", async () => {
+ it("filterOptions returns tags and evaluators with specific alerts", async () => {
const { project, caller } = await prepare();
+ const linkedEvaluatorId = v4();
+ const unlinkedEvaluatorId = v4();
+ await prisma.evaluator.createMany({
+ data: [
+ {
+ id: linkedEvaluatorId,
+ projectId: project.id,
+ name: "Linked evaluator",
+ type: "CODE",
+ },
+ {
+ id: unlinkedEvaluatorId,
+ projectId: project.id,
+ name: "Unlinked evaluator",
+ type: "CODE",
+ },
+ ],
+ });
await caller.monitors.create({
...validMonitorInput(project.id),
name: "A",
tags: ["prod", "latency"],
+ filters: [
+ {
+ column: "evaluatorId",
+ type: "string",
+ operator: "contains",
+ value: linkedEvaluatorId.slice(1, -1),
+ },
+ ],
});
await caller.monitors.create({
...validMonitorInput(project.id),
@@ -432,6 +579,25 @@ describe("monitors trpc", () => {
projectId: project.id,
});
expect(opts.tags.map((t) => t.value).sort()).toEqual(["latency", "prod"]);
+ expect(opts.evaluators).toEqual([
+ { value: linkedEvaluatorId, displayValue: "Linked evaluator" },
+ ]);
+
+ const filtered = await caller.monitors.all({
+ projectId: project.id,
+ orderBy: null,
+ page: 1,
+ limit: 50,
+ filter: [
+ {
+ type: "stringOptions",
+ column: "evaluatorId",
+ operator: "any of",
+ value: [linkedEvaluatorId],
+ },
+ ],
+ });
+ expect(filtered.monitors.map((monitor) => monitor.name)).toEqual(["A"]);
});
});
diff --git a/web/src/features/ai-features/server/index.ts b/web/src/features/ai-features/server/index.ts
new file mode 100644
index 0000000000..b6fc50ad56
--- /dev/null
+++ b/web/src/features/ai-features/server/index.ts
@@ -0,0 +1 @@
+export { resolveLangfuseAiFeatureAvailability } from "@/src/features/ai-features/server/availability";
diff --git a/web/src/features/dashboard/lib/dashboardUiTableToViewMapping.ts b/web/src/features/dashboard/lib/dashboardUiTableToViewMapping.ts
index b239bee115..2c9248e9e5 100644
--- a/web/src/features/dashboard/lib/dashboardUiTableToViewMapping.ts
+++ b/web/src/features/dashboard/lib/dashboardUiTableToViewMapping.ts
@@ -56,6 +56,23 @@ const defineField = (
legacy,
});
+const evaluatorFilterDefinitions = [
+ defineField(
+ "evaluatorId",
+ sourceSpec("Evaluator", {
+ uiTableId: "evaluatorId",
+ aliases: ["Evaluator ID"],
+ }),
+ ),
+ defineField(
+ "isEvaluatorTest",
+ sourceSpec("Evaluator test run", {
+ uiTableId: "isEvaluatorTest",
+ aliases: ["Evaluator execution"],
+ }),
+ ),
+] as const;
+
const viewFilterDefinitions: Record<
ViewName,
readonly DashboardViewFieldDefinition[]
@@ -145,6 +162,7 @@ const viewFilterDefinitions: Record<
"environment",
sourceSpec("Environment", { uiTableId: "environment" }),
),
+ ...evaluatorFilterDefinitions,
defineField(
"release",
sourceSpec("Release", {
@@ -220,6 +238,7 @@ const viewFilterDefinitions: Record<
"traceVersion",
sourceSpec("Version", { uiTableId: "version" }),
),
+ ...evaluatorFilterDefinitions,
],
"scores-boolean": [
defineField("name", sourceSpec("Score Name", { uiTableId: "scoreName" })),
@@ -267,6 +286,7 @@ const viewFilterDefinitions: Record<
"traceVersion",
sourceSpec("Version", { uiTableId: "version" }),
),
+ ...evaluatorFilterDefinitions,
],
"scores-categorical": [
defineField("name", sourceSpec("Score Name", { uiTableId: "scoreName" })),
@@ -314,6 +334,7 @@ const viewFilterDefinitions: Record<
"traceVersion",
sourceSpec("Version", { uiTableId: "version" }),
),
+ ...evaluatorFilterDefinitions,
],
};
diff --git a/web/src/features/entitlements/index.ts b/web/src/features/entitlements/index.ts
new file mode 100644
index 0000000000..172f4efcc2
--- /dev/null
+++ b/web/src/features/entitlements/index.ts
@@ -0,0 +1 @@
+export { useEntitlementLimit } from "@/src/features/entitlements/hooks";
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorAlertButton/EvaluatorAlertButton.clienttest.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorAlertButton/EvaluatorAlertButton.clienttest.tsx
new file mode 100644
index 0000000000..ff27460a19
--- /dev/null
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorAlertButton/EvaluatorAlertButton.clienttest.tsx
@@ -0,0 +1,191 @@
+import { fireEvent, render, screen } from "@testing-library/react";
+
+import { EvaluatorAlertButton } from "./EvaluatorAlertButton";
+
+const routerPush = vi.hoisted(() => vi.fn());
+
+vi.mock("next/router", () => ({
+ useRouter: () => ({ push: routerPush }),
+}));
+
+vi.mock("@/src/features/posthog-analytics", () => ({
+ usePostHogClientCapture: () => vi.fn(),
+}));
+
+describe("EvaluatorAlertButton", () => {
+ beforeAll(() => {
+ vi.stubGlobal(
+ "ResizeObserver",
+ class {
+ observe() {}
+ unobserve() {}
+ disconnect() {}
+ },
+ );
+ });
+
+ afterAll(() => {
+ vi.unstubAllGlobals();
+ });
+
+ beforeEach(() => {
+ routerPush.mockReset().mockResolvedValue(true);
+ });
+
+ it("offers to add an alert when none are configured", () => {
+ const { container } = render(
+ <EvaluatorAlertButton
+ scope="allEvaluators"
+ projectId="project-1"
+ connectedAlerts={[]}
+ canRead
+ canCreate
+ />,
+ );
+
+ expect(
+ screen.getByRole("button", { name: "Add evaluator alert" }),
+ ).toHaveTextContent("Add alert");
+ expect(container.querySelector(".lucide-plus")).toBeInTheDocument();
+ expect(container.querySelector(".lucide-bell")).not.toBeInTheDocument();
+ });
+
+ it("keeps the alert label and shows a spinner while loading", () => {
+ const { container } = render(
+ <EvaluatorAlertButton
+ scope="allEvaluators"
+ projectId="project-1"
+ connectedAlerts={[]}
+ isLoading
+ canRead
+ canCreate
+ />,
+ );
+
+ expect(
+ screen.getByRole("button", { name: "Loading evaluator alerts" }),
+ ).toHaveTextContent("Add alert");
+ expect(container.querySelector(".lucide-loader-circle")).toHaveClass(
+ "animate-spin",
+ );
+ expect(container.querySelector(".lucide-plus")).not.toBeInTheDocument();
+ });
+
+ it("uses the evaluator score style for code evaluators", () => {
+ render(
+ <EvaluatorAlertButton
+ scope="evaluator"
+ projectId="project-1"
+ evaluatorId="evaluator-1"
+ evaluatorType="CODE"
+ scoreDataType="BOOLEAN"
+ connectedAlerts={[]}
+ canRead
+ canCreate
+ />,
+ );
+
+ fireEvent.click(
+ screen.getByRole("button", { name: "Add evaluator alert" }),
+ );
+
+ const scoreAlert = screen
+ .getByText("Score threshold")
+ .closest("[cmdk-item]");
+ expect(scoreAlert).not.toBeNull();
+ expect(
+ scoreAlert?.querySelector(".lucide-arrow-up-right"),
+ ).toBeInTheDocument();
+ expect(screen.queryByText("Numeric score")).not.toBeInTheDocument();
+ expect(screen.queryByText("Boolean score")).not.toBeInTheDocument();
+ expect(screen.queryByText("Categorical score")).not.toBeInTheDocument();
+ expect(screen.queryByText("Cost threshold")).not.toBeInTheDocument();
+ });
+
+ it("does not offer another cost alert for connected code evaluators", () => {
+ render(
+ <EvaluatorAlertButton
+ scope="evaluator"
+ projectId="project-1"
+ evaluatorId="evaluator-1"
+ evaluatorType="CODE"
+ scoreDataType="BOOLEAN"
+ connectedAlerts={[
+ {
+ id: "alert-1",
+ name: "Evaluator alert",
+ status: "ACTIVE",
+ severity: "UNKNOWN",
+ metric: { measure: "count", aggregation: "count" },
+ thresholdOperator: "GT",
+ alertThreshold: 1,
+ alertedAt: null,
+ },
+ ]}
+ canRead
+ canCreate
+ />,
+ );
+
+ fireEvent.click(
+ screen.getByRole("button", { name: "1 connected evaluator alert" }),
+ );
+
+ expect(screen.getByRole("button", { name: "Score" })).toBeInTheDocument();
+ expect(
+ screen.queryByRole("button", { name: "Cost" }),
+ ).not.toBeInTheDocument();
+ });
+
+ it("shows a table link after 20 connected alerts and keeps bottom actions icon-free", () => {
+ render(
+ <EvaluatorAlertButton
+ scope="evaluator"
+ projectId="project-1"
+ evaluatorId="evaluator-1"
+ evaluatorType="LLM_AS_JUDGE"
+ scoreDataType="NUMERIC"
+ connectedAlerts={[
+ {
+ id: "alert-1",
+ name: "Evaluator alert",
+ status: "ACTIVE",
+ severity: "UNKNOWN",
+ metric: { measure: "count", aggregation: "count" },
+ thresholdOperator: "GT",
+ alertThreshold: 1,
+ alertedAt: null,
+ },
+ ]}
+ hasMore
+ canRead
+ canCreate
+ />,
+ );
+
+ fireEvent.click(
+ screen.getByRole("button", { name: "1 connected evaluator alert" }),
+ );
+
+ const connectedAlert = screen
+ .getByText("Evaluator alert")
+ .closest("[cmdk-item]");
+ expect(
+ connectedAlert?.querySelector(".lucide-arrow-up-right"),
+ ).toBeInTheDocument();
+
+ const scoreAction = screen.getByRole("button", { name: "Score" });
+ const costAction = screen.getByRole("button", { name: "Cost" });
+ expect(
+ scoreAction.querySelector(".lucide-arrow-up-right"),
+ ).not.toBeInTheDocument();
+ expect(
+ costAction.querySelector(".lucide-arrow-up-right"),
+ ).not.toBeInTheDocument();
+
+ fireEvent.click(screen.getByText("See all alerts"));
+ expect(routerPush).toHaveBeenCalledWith(
+ "/project/project-1/alerts?filter=evaluatorId%3BstringOptions%3B%3Bany+of%3Bevaluator-1",
+ );
+ });
+});
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorAlertButton/EvaluatorAlertButton.stories.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorAlertButton/EvaluatorAlertButton.stories.tsx
new file mode 100644
index 0000000000..dc5f076c9e
--- /dev/null
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorAlertButton/EvaluatorAlertButton.stories.tsx
@@ -0,0 +1,116 @@
+import preview from "../../../../../../../.storybook/preview";
+import { EvaluatorAlertButton } from "./EvaluatorAlertButton";
+
+const meta = preview.meta({ component: EvaluatorAlertButton });
+
+const alertDefaults = {
+ status: "ACTIVE",
+ metric: { measure: "value", aggregation: "avg" },
+ thresholdOperator: "LT",
+ alertThreshold: 0.7,
+ alertedAt: new Date(Date.now() - 2 * 24 * 60 * 60 * 1000),
+} as const;
+
+const defaultArgs = {
+ projectId: "project-1",
+ scope: "evaluator" as const,
+ evaluatorId: "evaluator-1",
+ evaluatorType: "LLM_AS_JUDGE" as const,
+ scoreDataType: "NUMERIC" as const,
+ connectedAlerts: [],
+ canRead: true,
+ canCreate: true,
+};
+
+export const NoAlerts = meta.story({ args: defaultArgs });
+
+export const ConnectedAlerts = meta.story({
+ args: {
+ ...defaultArgs,
+ connectedAlerts: [
+ {
+ ...alertDefaults,
+ id: "alert-1",
+ name: "Accuracy dropped",
+ severity: "ALERT",
+ },
+ {
+ ...alertDefaults,
+ id: "alert-2",
+ name: "Evaluator spend increased",
+ severity: "WARNING",
+ metric: { measure: "totalCost", aggregation: "sum" },
+ thresholdOperator: "GT",
+ alertThreshold: 10,
+ },
+ {
+ ...alertDefaults,
+ id: "alert-3",
+ name: "Low quality score",
+ severity: "OK",
+ },
+ ],
+ },
+});
+
+export const CodeEvaluator = meta.story({
+ args: {
+ ...defaultArgs,
+ evaluatorType: "CODE",
+ scoreDataType: "BOOLEAN",
+ },
+});
+
+export const LoadingSpinner = meta.story({
+ args: { ...defaultArgs, isLoading: true },
+});
+
+export const ReadOnly = meta.story({
+ args: {
+ ...defaultArgs,
+ connectedAlerts: [
+ {
+ ...alertDefaults,
+ id: "alert-1",
+ name: "Accuracy dropped",
+ status: "PAUSED",
+ severity: "PAUSED",
+ },
+ ],
+ canCreate: false,
+ },
+});
+
+export const LimitReached = meta.story({
+ args: { ...defaultArgs, limitReached: true },
+});
+
+export const AggregateCost = meta.story({
+ args: {
+ projectId: "project-1",
+ scope: "allEvaluators",
+ connectedAlerts: [],
+ canRead: true,
+ canCreate: true,
+ },
+});
+
+export const AggregateCostConnected = meta.story({
+ args: {
+ projectId: "project-1",
+ scope: "allEvaluators",
+ connectedAlerts: [
+ {
+ ...alertDefaults,
+ id: "alert-1",
+ name: "Weekly evaluator cost",
+ severity: "OK",
+ metric: { measure: "totalCost", aggregation: "sum" },
+ thresholdOperator: "GT",
+ alertThreshold: 50,
+ },
+ ],
+ canRead: true,
+ canCreate: true,
+ },
+});
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorAlertButton/EvaluatorAlertButton.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorAlertButton/EvaluatorAlertButton.tsx
new file mode 100644
index 0000000000..0c153a20d9
--- /dev/null
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorAlertButton/EvaluatorAlertButton.tsx
@@ -0,0 +1,339 @@
+import { useRouter } from "next/router";
+import {
+ ArrowUpRight,
+ Bell,
+ ChevronDown,
+ DollarSign,
+ Gauge,
+ LoaderCircle,
+ Plus,
+} from "lucide-react";
+
+import { Badge } from "@/src/components/ui/badge";
+import { Button } from "@/src/components/ui/button";
+import {
+ Command,
+ CommandGroup,
+ CommandItem,
+ CommandList,
+ CommandSeparator,
+} from "@/src/components/ui/command";
+import { PopoverController } from "@/src/components/ui/popover";
+import {
+ evaluatorAlertsListUrl,
+ evaluatorAlertUrl,
+} from "@/src/features/evals/v2/fns/evaluators/evaluatorAlertUrl";
+import { MonitorSeverityBadge } from "@/src/features/monitors";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics";
+import { renderEvaluatorAlertLastFired } from "@/src/features/evals/v2/fns/evaluators/renderEvaluatorAlertLastFired";
+import { renderEvaluatorAlertTriggerCondition } from "@/src/features/evals/v2/fns/evaluators/renderEvaluatorAlertTriggerCondition";
+import type { RouterOutputs } from "@/src/utils/api";
+
+type ConnectedAlert =
+ RouterOutputs["monitors"]["linkedEvaluatorAlerts"]["data"][number];
+
+type EvaluatorAlertButtonProps = {
+ projectId: string;
+ connectedAlerts: ConnectedAlert[];
+ hasMore?: boolean;
+ isLoading?: boolean;
+ canRead: boolean;
+ canCreate: boolean;
+ limitReached?: boolean;
+} & (
+ | {
+ scope: "evaluator";
+ evaluatorId: string;
+ evaluatorType: "LLM_AS_JUDGE" | "CODE";
+ scoreDataType?: "NUMERIC" | "BOOLEAN" | "CATEGORICAL";
+ }
+ | { scope: "allEvaluators" }
+);
+
+/** Alert picker for one evaluator or aggregate evaluator cost. */
+export function EvaluatorAlertButton(props: EvaluatorAlertButtonProps) {
+ const {
+ projectId,
+ connectedAlerts,
+ hasMore = false,
+ isLoading = false,
+ canRead,
+ canCreate,
+ limitReached = false,
+ } = props;
+ const router = useRouter();
+ const capture = usePostHogClientCapture();
+ const createDisabled = !canCreate || limitReached;
+ const disabledReason = limitReached
+ ? "Alert limit reached"
+ : "You do not have permission to create alerts";
+ const alertCount = connectedAlerts.length;
+ const isAggregateCost = props.scope === "allEvaluators";
+ const supportsCostAlert =
+ isAggregateCost ||
+ (props.scope === "evaluator" && props.evaluatorType === "LLM_AS_JUDGE");
+ const scoreAlertUrl = (dataType: "NUMERIC" | "BOOLEAN" | "CATEGORICAL") => {
+ if (props.scope !== "evaluator") return "";
+ return evaluatorAlertUrl(projectId, {
+ type: "score",
+ evaluatorId: props.evaluatorId,
+ scoreDataType: dataType,
+ });
+ };
+ const costAlertUrl = evaluatorAlertUrl(
+ projectId,
+ isAggregateCost
+ ? { type: "allEvaluatorCost" }
+ : { type: "cost", evaluatorId: props.evaluatorId },
+ );
+ const selectedScoreDataType =
+ props.scope === "evaluator" ? props.scoreDataType : undefined;
+ const navigate = (href: string, closePopover: () => void) => {
+ closePopover();
+ router.push(href).catch(() => undefined);
+ };
+ const openAlertCreation = (
+ alertType: "score" | "cost" | "all_evaluator_cost",
+ href: string,
+ closePopover: () => void,
+ dataType?: "NUMERIC" | "BOOLEAN" | "CATEGORICAL",
+ ) => {
+ capture("evaluators:alert_create_clicked", {
+ source: isAggregateCost ? "evaluator_overview" : "evaluator_detail",
+ alertType,
+ scoreDataType: dataType,
+ });
+ navigate(href, closePopover);
+ };
+
+ return (
+ <PopoverController
+ align="end"
+ contentClassName="w-96 p-0"
+ disabled={false}
+ modal={false}
+ renderContent={({ closePopover }) => (
+ <Command defaultValue="__none__">
+ <CommandList>
+ {canRead && connectedAlerts.length > 0 ? (
+ <div className="p-2">
+ <p className="text-muted-foreground px-2 py-1.5 text-[10px] font-bold tracking-wider uppercase">
+ Connected · {connectedAlerts.length}
+ {hasMore ? "+" : ""}
+ </p>
+ <div className="space-y-1">
+ {connectedAlerts.map((alert) => (
+ <CommandItem
+ key={alert.id}
+ value={`connected ${alert.name} ${alert.id}`}
+ className="items-start rounded-md px-3 py-2.5"
+ onSelect={() =>
+ navigate(
+ `/project/${projectId}/alerts/${alert.id}`,
+ closePopover,
+ )
+ }
+ >
+ <div className="min-w-0 flex-1">
+ <p className="truncate font-bold" title={alert.name}>
+ {alert.name}
+ </p>
+ <p className="text-muted-foreground mt-0.5 text-xs">
+ {renderEvaluatorAlertTriggerCondition(alert)}
+ <span className="px-1" aria-hidden="true">
+ ·
+ </span>
+ {renderEvaluatorAlertLastFired(alert.alertedAt)}
+ </p>
+ </div>
+ <MonitorSeverityBadge
+ severity={alert.severity}
+ className="ml-auto w-auto shrink-0 px-2 py-0.5 text-[10px]"
+ />
+ <ArrowUpRight className="text-muted-foreground h-4 w-4 shrink-0" />
+ </CommandItem>
+ ))}
+ </div>
+ {hasMore ? (
+ <CommandItem
+ value="see all alerts"
+ className="mt-1"
+ onSelect={() =>
+ navigate(
+ evaluatorAlertsListUrl(
+ projectId,
+ props.scope === "evaluator"
+ ? props.evaluatorId
+ : undefined,
+ ),
+ closePopover,
+ )
+ }
+ >
+ <span>See all alerts</span>
+ <ArrowUpRight className="text-muted-foreground ml-auto h-4 w-4 shrink-0" />
+ </CommandItem>
+ ) : null}
+ </div>
+ ) : (
+ <div className="px-4 py-3">
+ <p className="text-sm font-bold">
+ {isAggregateCost
+ ? "No alerts on evaluator cost"
+ : "No alerts on this evaluator"}
+ </p>
+ <p className="text-muted-foreground mt-0.5 text-xs">
+ {isAggregateCost
+ ? "Get notified when evaluator cost crosses a threshold."
+ : supportsCostAlert
+ ? "Get notified when this evaluator's scores or cost cross a threshold."
+ : "Get notified when this evaluator's score crosses a threshold."}
+ </p>
+ </div>
+ )}
+ <CommandSeparator className="mx-0" />
+ {connectedAlerts.length > 0 ? (
+ <div className="flex items-center justify-between gap-3 p-2">
+ <span className="text-muted-foreground flex h-8 items-center px-1 text-sm leading-none">
+ Add alert
+ </span>
+ <div className="flex flex-wrap items-center justify-end gap-1.5">
+ {selectedScoreDataType ? (
+ <Button
+ type="button"
+ variant="outline"
+ size="sm"
+ className="h-8 cursor-pointer gap-1.5"
+ disabled={createDisabled}
+ onClick={() =>
+ openAlertCreation(
+ "score",
+ scoreAlertUrl(selectedScoreDataType),
+ closePopover,
+ selectedScoreDataType,
+ )
+ }
+ >
+ <Gauge className="text-muted-foreground h-4 w-4" />
+ Score
+ </Button>
+ ) : null}
+ {supportsCostAlert ? (
+ <Button
+ type="button"
+ variant="outline"
+ size="sm"
+ className="h-8 cursor-pointer gap-1.5"
+ disabled={createDisabled}
+ onClick={() =>
+ openAlertCreation(
+ isAggregateCost ? "all_evaluator_cost" : "cost",
+ costAlertUrl,
+ closePopover,
+ )
+ }
+ >
+ <DollarSign className="text-muted-foreground h-4 w-4" />
+ Cost
+ </Button>
+ ) : null}
+ </div>
+ </div>
+ ) : (
+ <CommandGroup>
+ {selectedScoreDataType ? (
+ <CommandItem
+ disabled={createDisabled}
+ onSelect={() =>
+ openAlertCreation(
+ "score",
+ scoreAlertUrl(selectedScoreDataType),
+ closePopover,
+ selectedScoreDataType,
+ )
+ }
+ >
+ <Gauge className="text-muted-foreground" />
+ <div>
+ <p>Score threshold</p>
+ <p className="text-muted-foreground text-xs">
+ Alert when this evaluator&apos;s score crosses a
+ threshold
+ </p>
+ </div>
+ <ArrowUpRight className="text-muted-foreground ml-auto h-4 w-4 shrink-0" />
+ </CommandItem>
+ ) : null}
+ {supportsCostAlert ? (
+ <CommandItem
+ disabled={createDisabled}
+ onSelect={() =>
+ openAlertCreation(
+ isAggregateCost ? "all_evaluator_cost" : "cost",
+ costAlertUrl,
+ closePopover,
+ )
+ }
+ >
+ <DollarSign className="text-muted-foreground" />
+ <div>
+ <p>Cost threshold</p>
+ <p className="text-muted-foreground text-xs">
+ {isAggregateCost
+ ? "Alert when evaluator cost crosses a threshold"
+ : "Alert on spend from running this evaluator"}
+ </p>
+ </div>
+ <ArrowUpRight className="text-muted-foreground ml-auto h-4 w-4 shrink-0" />
+ </CommandItem>
+ ) : null}
+ </CommandGroup>
+ )}
+ {createDisabled ? (
+ <p className="text-muted-foreground px-3 pb-2 text-xs">
+ {disabledReason}
+ </p>
+ ) : null}
+ </CommandList>
+ </Command>
+ )}
+ >
+ {({ Trigger }) => (
+ <Trigger asChild>
+ <Button
+ type="button"
+ variant="outline"
+ aria-label={
+ isLoading
+ ? "Loading evaluator alerts"
+ : alertCount > 0
+ ? `${alertCount} connected evaluator alert${alertCount === 1 ? "" : "s"}`
+ : "Add evaluator alert"
+ }
+ className="w-auto max-w-full justify-start"
+ >
+ {isLoading ? (
+ <LoaderCircle
+ className="mr-1 h-4 w-4 shrink-0 animate-spin"
+ aria-hidden="true"
+ />
+ ) : alertCount > 0 ? (
+ <Bell className="mr-1 h-4 w-4 shrink-0" aria-hidden="true" />
+ ) : (
+ <Plus className="mr-1 h-4 w-4 shrink-0" aria-hidden="true" />
+ )}
+ <span className="flex-1 text-left">
+ {alertCount > 0 ? "Alerts" : "Add alert"}
+ </span>
+ {!isLoading && alertCount > 0 ? (
+ <Badge variant="secondary" size="sm" className="ml-1 shrink-0">
+ {alertCount}
+ </Badge>
+ ) : null}
+ <ChevronDown className="h-4 w-4 shrink-0 opacity-50" />
+ </Button>
+ </Trigger>
+ )}
+ </PopoverController>
+ );
+}
diff --git a/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker.tsx b/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker.tsx
index c1ac378533..04e4e138b9 100644
--- a/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker.tsx
@@ -68,6 +68,7 @@ type JudgeModelPickerTriggerProps = Omit<
selectedModel: JudgeModel | null;
disabled: boolean;
missingDefaultLabel?: string;
+ borderVariant?: "default" | "contrast";
};
export const JudgeModelPickerTrigger = forwardRef<
@@ -86,6 +87,7 @@ export const JudgeModelPickerTrigger = forwardRef<
loading,
loadingText,
disabled,
+ borderVariant = "default",
...buttonProps
},
forwardedRef,
@@ -114,6 +116,7 @@ export const JudgeModelPickerTrigger = forwardRef<
className={cn(
selectTriggerClassName,
"w-auto max-w-full min-w-0 justify-start",
+ borderVariant === "contrast" && "border-border-contrast",
)}
>
{mode === "default" ? (
diff --git a/web/src/features/evals/v2/components/VariableMapping/VariableMapping.tsx b/web/src/features/evals/v2/components/VariableMapping/VariableMapping.tsx
index d5e925db16..f4e2451b12 100644
--- a/web/src/features/evals/v2/components/VariableMapping/VariableMapping.tsx
+++ b/web/src/features/evals/v2/components/VariableMapping/VariableMapping.tsx
@@ -52,7 +52,7 @@ function ReadOnlyVariableMapping({
function EmptyVariableMapping({ mode }: Pick<VariableMappingProps, "mode">) {
return (
- <div className="border-border bg-muted/20 flex flex-col items-center justify-center gap-2 rounded-md border border-dashed px-6 py-8 text-center">
+ <div className="border-border bg-muted/20 flex flex-col items-center justify-center gap-2 rounded-md border px-6 py-8 text-center">
<div className="bg-muted text-muted-foreground flex size-9 items-center justify-center rounded-full">
<Braces className="size-4" aria-hidden />
</div>
diff --git a/web/src/features/evals/v2/fns/evaluators/evaluatorAlertUrl.clienttest.ts b/web/src/features/evals/v2/fns/evaluators/evaluatorAlertUrl.clienttest.ts
new file mode 100644
index 0000000000..d4bc41698e
--- /dev/null
+++ b/web/src/features/evals/v2/fns/evaluators/evaluatorAlertUrl.clienttest.ts
@@ -0,0 +1,48 @@
+import { describe, expect, it } from "vitest";
+
+import {
+ evaluatorAlertsListUrl,
+ evaluatorAlertUrl,
+} from "@/src/features/evals/v2/fns/evaluators/evaluatorAlertUrl";
+
+describe("evaluatorAlertUrl", () => {
+ it("builds evaluator-specific score and cost alert URLs", () => {
+ expect(
+ evaluatorAlertUrl("project/1", {
+ type: "score",
+ evaluatorId: "eval/1",
+ scoreDataType: "BOOLEAN",
+ }),
+ ).toBe(
+ "/project/project%2F1/alerts/new?alert=evaluatorScore&evaluatorId=eval%2F1&scoreDataType=BOOLEAN",
+ );
+ expect(
+ evaluatorAlertUrl("project/1", {
+ type: "cost",
+ evaluatorId: "eval/1",
+ }),
+ ).toBe(
+ "/project/project%2F1/alerts/new?alert=evaluatorCost&evaluatorId=eval%2F1",
+ );
+ });
+
+ it("builds the all-evaluator spend URL", () => {
+ expect(evaluatorAlertUrl("project/1", { type: "allEvaluatorCost" })).toBe(
+ "/project/project%2F1/alerts/new?alert=allEvaluatorCost",
+ );
+ });
+});
+
+describe("evaluatorAlertsListUrl", () => {
+ it("filters evaluator-specific alerts by evaluator ID", () => {
+ expect(evaluatorAlertsListUrl("project/1", "eval/1")).toBe(
+ "/project/project%2F1/alerts?filter=evaluatorId%3BstringOptions%3B%3Bany+of%3Beval%252F1",
+ );
+ });
+
+ it("leaves the all-evaluator alerts list unfiltered", () => {
+ expect(evaluatorAlertsListUrl("project/1")).toBe(
+ "/project/project%2F1/alerts",
+ );
+ });
+});
diff --git a/web/src/features/evals/v2/fns/evaluators/evaluatorAlertUrl.ts b/web/src/features/evals/v2/fns/evaluators/evaluatorAlertUrl.ts
new file mode 100644
index 0000000000..7d046843b7
--- /dev/null
+++ b/web/src/features/evals/v2/fns/evaluators/evaluatorAlertUrl.ts
@@ -0,0 +1,50 @@
+import { encodeFiltersGeneric } from "@langfuse/shared";
+
+/** Builds an alerts-list URL, optionally filtered to one evaluator. */
+export function evaluatorAlertsListUrl(
+ projectId: string,
+ evaluatorId?: string,
+): string {
+ const base = `/project/${encodeURIComponent(projectId)}/alerts`;
+ if (!evaluatorId) return base;
+
+ const filter = encodeFiltersGeneric([
+ {
+ column: "evaluatorId",
+ type: "stringOptions",
+ operator: "any of",
+ value: [evaluatorId],
+ },
+ ]);
+ const params = new URLSearchParams({ filter });
+ return `${base}?${params.toString()}`;
+}
+
+/** Builds a new-alert URL using only evaluator alert prefill fields. */
+export function evaluatorAlertUrl(
+ projectId: string,
+ alert:
+ | {
+ type: "score";
+ evaluatorId: string;
+ scoreDataType: "NUMERIC" | "BOOLEAN" | "CATEGORICAL";
+ }
+ | { type: "cost"; evaluatorId: string }
+ | { type: "allEvaluatorCost" },
+): string {
+ const base = `/project/${encodeURIComponent(projectId)}/alerts/new`;
+ const params = new URLSearchParams();
+
+ if (alert.type === "allEvaluatorCost") {
+ params.set("alert", "allEvaluatorCost");
+ } else if (alert.type === "cost") {
+ params.set("alert", "evaluatorCost");
+ params.set("evaluatorId", alert.evaluatorId);
+ } else {
+ params.set("alert", "evaluatorScore");
+ params.set("evaluatorId", alert.evaluatorId);
+ params.set("scoreDataType", alert.scoreDataType);
+ }
+
+ return `${base}?${params.toString()}`;
+}
diff --git a/web/src/features/evals/v2/fns/evaluators/getFirstCodeEvaluatorScoreDataType.clienttest.ts b/web/src/features/evals/v2/fns/evaluators/getFirstCodeEvaluatorScoreDataType.clienttest.ts
new file mode 100644
index 0000000000..5bee85f703
--- /dev/null
+++ b/web/src/features/evals/v2/fns/evaluators/getFirstCodeEvaluatorScoreDataType.clienttest.ts
@@ -0,0 +1,41 @@
+import { getFirstCodeEvaluatorScoreDataType } from "./getFirstCodeEvaluatorScoreDataType";
+
+describe("getFirstCodeEvaluatorScoreDataType", () => {
+ it("uses the first supported score data type in TypeScript source", () => {
+ expect(
+ getFirstCodeEvaluatorScoreDataType(`
+ return {
+ scores: [
+ { name: "Passed", dataType: "BOOLEAN", value: true },
+ { name: "Confidence", dataType: "NUMERIC", value: 1 },
+ ],
+ };
+ `),
+ ).toBe("BOOLEAN");
+ });
+
+ it("reads quoted Python dictionary keys", () => {
+ expect(
+ getFirstCodeEvaluatorScoreDataType(`
+ return {
+ "scores": [
+ {"name": "Confidence", "dataType": "NUMERIC", "value": 1}
+ ]
+ }
+ `),
+ ).toBe("NUMERIC");
+ });
+
+ it("does not skip an unsupported first score type", () => {
+ expect(
+ getFirstCodeEvaluatorScoreDataType(`
+ return {
+ scores: [
+ { name: "Explanation", dataType: "TEXT", value: "ok" },
+ { name: "Confidence", dataType: "NUMERIC", value: 1 },
+ ],
+ };
+ `),
+ ).toBeUndefined();
+ });
+});
diff --git a/web/src/features/evals/v2/fns/evaluators/getFirstCodeEvaluatorScoreDataType.ts b/web/src/features/evals/v2/fns/evaluators/getFirstCodeEvaluatorScoreDataType.ts
new file mode 100644
index 0000000000..b24460feef
--- /dev/null
+++ b/web/src/features/evals/v2/fns/evaluators/getFirstCodeEvaluatorScoreDataType.ts
@@ -0,0 +1,21 @@
+export type AlertScoreDataType = "NUMERIC" | "BOOLEAN" | "CATEGORICAL";
+
+const ALERT_SCORE_DATA_TYPES = new Set<AlertScoreDataType>([
+ "NUMERIC",
+ "BOOLEAN",
+ "CATEGORICAL",
+]);
+
+/** Reads the first score data type declared in stored code evaluator source. */
+export function getFirstCodeEvaluatorScoreDataType(
+ sourceCode: string,
+): AlertScoreDataType | undefined {
+ const match = sourceCode.match(
+ /(?:["']dataType["']|dataType)\s*:\s*(?:["'](NUMERIC|BOOLEAN|CATEGORICAL|TEXT)["']|ScoreDataTypeEnum\.(NUMERIC|BOOLEAN|CATEGORICAL|TEXT))/,
+ );
+ const dataType = match?.[1] ?? match?.[2];
+
+ return dataType && ALERT_SCORE_DATA_TYPES.has(dataType as AlertScoreDataType)
+ ? (dataType as AlertScoreDataType)
+ : undefined;
+}
diff --git a/web/src/features/evals/v2/fns/evaluators/renderEvaluatorAlertLastFired.clienttest.ts b/web/src/features/evals/v2/fns/evaluators/renderEvaluatorAlertLastFired.clienttest.ts
new file mode 100644
index 0000000000..83efde8fab
--- /dev/null
+++ b/web/src/features/evals/v2/fns/evaluators/renderEvaluatorAlertLastFired.clienttest.ts
@@ -0,0 +1,24 @@
+// @vitest-environment node
+
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+import { renderEvaluatorAlertLastFired } from "./renderEvaluatorAlertLastFired";
+
+describe("renderEvaluatorAlertLastFired", () => {
+ beforeEach(() => {
+ vi.useFakeTimers();
+ vi.setSystemTime(new Date("2026-06-03T12:00:00.000Z"));
+ });
+
+ afterEach(() => vi.useRealTimers());
+
+ it("renders the relative time of the latest alert emission", () => {
+ expect(
+ renderEvaluatorAlertLastFired(new Date("2026-06-01T12:00:00.000Z")),
+ ).toBe("Last fired 2 days ago");
+ });
+
+ it("renders alerts that have not fired", () => {
+ expect(renderEvaluatorAlertLastFired(null)).toBe("Never fired");
+ });
+});
diff --git a/web/src/features/evals/v2/fns/evaluators/renderEvaluatorAlertLastFired.ts b/web/src/features/evals/v2/fns/evaluators/renderEvaluatorAlertLastFired.ts
new file mode 100644
index 0000000000..b6d4940a64
--- /dev/null
+++ b/web/src/features/evals/v2/fns/evaluators/renderEvaluatorAlertLastFired.ts
@@ -0,0 +1,8 @@
+import { formatDistanceToNowStrict } from "date-fns";
+
+/** Renders the most recent evaluator alert emission time. */
+export function renderEvaluatorAlertLastFired(alertedAt: Date | null): string {
+ return alertedAt
+ ? `Last fired ${formatDistanceToNowStrict(alertedAt, { addSuffix: true })}`
+ : "Never fired";
+}
diff --git a/web/src/features/evals/v2/fns/evaluators/renderEvaluatorAlertTriggerCondition.clienttest.ts b/web/src/features/evals/v2/fns/evaluators/renderEvaluatorAlertTriggerCondition.clienttest.ts
new file mode 100644
index 0000000000..5830b21738
--- /dev/null
+++ b/web/src/features/evals/v2/fns/evaluators/renderEvaluatorAlertTriggerCondition.clienttest.ts
@@ -0,0 +1,27 @@
+// @vitest-environment node
+
+import { describe, expect, it } from "vitest";
+
+import { renderEvaluatorAlertTriggerCondition } from "./renderEvaluatorAlertTriggerCondition";
+
+describe("renderEvaluatorAlertTriggerCondition", () => {
+ it("renders a count threshold", () => {
+ expect(
+ renderEvaluatorAlertTriggerCondition({
+ metric: { measure: "count", aggregation: "count" },
+ thresholdOperator: "GT",
+ alertThreshold: 5,
+ }),
+ ).toBe("Count > 5");
+ });
+
+ it("renders an aggregated measure threshold", () => {
+ expect(
+ renderEvaluatorAlertTriggerCondition({
+ metric: { measure: "value", aggregation: "avg" },
+ thresholdOperator: "LTE",
+ alertThreshold: 0.5,
+ }),
+ ).toBe("Avg value ≤ 0.5");
+ });
+});
diff --git a/web/src/features/evals/v2/fns/evaluators/renderEvaluatorAlertTriggerCondition.ts b/web/src/features/evals/v2/fns/evaluators/renderEvaluatorAlertTriggerCondition.ts
new file mode 100644
index 0000000000..63c3dcca8b
--- /dev/null
+++ b/web/src/features/evals/v2/fns/evaluators/renderEvaluatorAlertTriggerCondition.ts
@@ -0,0 +1,37 @@
+import { startCase } from "lodash";
+import type { z } from "zod";
+
+import { type metricAggregations } from "@langfuse/shared";
+import type { MonitorThresholdOperator } from "@langfuse/shared/monitors";
+
+const operatorSymbol: Record<MonitorThresholdOperator, string> = {
+ GT: ">",
+ GTE: "≥",
+ LT: "<",
+ LTE: "≤",
+ EQ: "=",
+ NEQ: "≠",
+};
+
+const aggregationLabel = (
+ aggregation: z.infer<typeof metricAggregations>,
+): string =>
+ /^p\d+$/.test(aggregation) ? aggregation : startCase(aggregation);
+
+/** Renders an evaluator alert boundary, e.g. "Count > 5". */
+export function renderEvaluatorAlertTriggerCondition({
+ metric,
+ thresholdOperator,
+ alertThreshold,
+}: {
+ metric: { measure: string; aggregation: z.infer<typeof metricAggregations> };
+ thresholdOperator: MonitorThresholdOperator;
+ alertThreshold: number;
+}): string {
+ const metricLabel =
+ metric.measure === "count"
+ ? "Count"
+ : `${aggregationLabel(metric.aggregation)} ${startCase(metric.measure).toLowerCase()}`;
+
+ return `${metricLabel} ${operatorSymbol[thresholdOperator]} ${alertThreshold}`;
+}
diff --git a/web/src/features/evals/v2/hooks/useEvaluatorAlerts.ts b/web/src/features/evals/v2/hooks/useEvaluatorAlerts.ts
new file mode 100644
index 0000000000..1cbc865778
--- /dev/null
+++ b/web/src/features/evals/v2/hooks/useEvaluatorAlerts.ts
@@ -0,0 +1,56 @@
+import { useEntitlementLimit } from "@/src/features/entitlements";
+import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
+import { api } from "@/src/utils/api";
+
+/** Owns evaluator alert permissions, linked-alert loading, and creation limits. */
+export function useEvaluatorAlerts(
+ params:
+ | { scope: "evaluator"; projectId: string; evaluatorId: string | null }
+ | { scope: "allEvaluators"; projectId: string },
+) {
+ const canRead = useHasProjectAccess({
+ projectId: params.projectId,
+ scope: "alerts:read",
+ });
+ const canCreate = useHasProjectAccess({
+ projectId: params.projectId,
+ scope: "alerts:CUD",
+ });
+ const monitorEntitlementLimit = useEntitlementLimit("monitor-count");
+ const evaluatorId =
+ params.scope === "evaluator" ? (params.evaluatorId ?? "") : "";
+
+ const evaluatorAlerts = api.monitors.linkedEvaluatorAlerts.useQuery(
+ { projectId: params.projectId, evaluatorId },
+ {
+ enabled: Boolean(params.scope === "evaluator" && evaluatorId && canRead),
+ },
+ );
+ const aggregateCostAlerts =
+ api.monitors.linkedAllEvaluatorSpendAlerts.useQuery(
+ { projectId: params.projectId },
+ {
+ enabled: Boolean(
+ params.scope === "allEvaluators" && params.projectId && canRead,
+ ),
+ },
+ );
+ const monitorCount = api.monitors.count.useQuery(
+ { projectId: params.projectId },
+ { enabled: Boolean(params.projectId && canCreate) },
+ );
+ const limitReached =
+ typeof monitorEntitlementLimit === "number" &&
+ (monitorCount.data?.count ?? 0) >= monitorEntitlementLimit;
+ const linkedAlerts =
+ params.scope === "evaluator" ? evaluatorAlerts : aggregateCostAlerts;
+
+ return {
+ connectedAlerts: linkedAlerts.data?.data ?? [],
+ hasMore: linkedAlerts.data?.hasMore ?? false,
+ isLoading: canRead && linkedAlerts.isPending,
+ canRead,
+ canCreate,
+ limitReached,
+ };
+}
diff --git a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
index 60785fbbf9..1f66f216c3 100644
--- a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
+++ b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
@@ -51,7 +51,11 @@ import { EvaluatorBlockedBanner } from "@/src/features/evals/v2/components/Evalu
import { useIsMobile } from "@/src/hooks/use-mobile";
import { prepareEvaluatorMetadataForSave } from "@/src/features/evals/v2/fns/prepareEvaluatorMetadataForSave";
import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
+import { useEvaluatorAlerts } from "@/src/features/evals/v2/hooks/useEvaluatorAlerts";
import { useCodeEvalSourceValidation } from "@/src/features/evals/hooks/useCodeEvalSourceValidation";
+import { EvaluatorAlertButton } from "@/src/features/evals/v2/components/Evaluators/EvaluatorAlertButton/EvaluatorAlertButton";
+import { toScoreOutputFormState } from "@/src/features/evals/v2/fns/scoreOutput/toScoreOutputFormState";
+import { getFirstCodeEvaluatorScoreDataType } from "@/src/features/evals/v2/fns/evaluators/getFirstCodeEvaluatorScoreDataType";
import {
getEvaluatorCreationAnalyticsProperties,
getJudgePromptAnalyticsProperties,
@@ -164,6 +168,19 @@ export function EvaluatorSetupPage(
projectId,
scope: "evaluator:CUD",
});
+ const evaluatorAlerts = useEvaluatorAlerts({
+ scope: "evaluator",
+ projectId,
+ evaluatorId: initialEvaluator?.id ?? null,
+ });
+ const scoreDataType = initialEvaluator
+ ? initialEvaluator.definition.type === "LLM_AS_JUDGE"
+ ? toScoreOutputFormState(initialEvaluator.definition.outputDefinition)
+ .dataType
+ : getFirstCodeEvaluatorScoreDataType(
+ initialEvaluator.definition.sourceCode,
+ )
+ : undefined;
const projectDefaultModel = useProjectDefaultModel({
projectId,
source: "editor",
@@ -677,6 +694,14 @@ export function EvaluatorSetupPage(
initialEvaluator.definition.variableMapping
}
/>
+ <EvaluatorAlertButton
+ scope="evaluator"
+ projectId={projectId}
+ evaluatorId={initialEvaluator.id}
+ evaluatorType={initialEvaluator.type}
+ scoreDataType={scoreDataType}
+ {...evaluatorAlerts}
+ />
<Button
type="button"
variant="outline"
diff --git a/web/src/features/evals/v2/pages/EvaluatorsPage.tsx b/web/src/features/evals/v2/pages/EvaluatorsPage.tsx
index f7626a24cb..b3ef4b696b 100644
--- a/web/src/features/evals/v2/pages/EvaluatorsPage.tsx
+++ b/web/src/features/evals/v2/pages/EvaluatorsPage.tsx
@@ -36,6 +36,7 @@ import { usePostHogClientCapture } from "@/src/features/posthog-analytics";
import useColumnOrder from "@/src/features/column-visibility/hooks/useColumnOrder";
import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
+import { useEvaluatorAlerts } from "@/src/features/evals/v2/hooks/useEvaluatorAlerts";
import { TableSelectionManager } from "@/src/features/table/components/TableSelectionManager";
import { usePaginationState } from "@/src/hooks/usePaginationState";
import { useSidebarFilterState } from "@/src/features/filters/hooks/useSidebarFilterState";
@@ -72,6 +73,7 @@ import { V4MigrationUpdateRequiredBadge } from "@/src/features/v4-migration/V4Mi
import { createNumberTableColumn } from "@/src/components/design-system/table/columns/createNumberTableColumn";
import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
import { createUserTableColumn } from "@/src/components/design-system/table/columns/createUserTableColumn";
+import { EvaluatorAlertButton } from "@/src/features/evals/v2/components/Evaluators/EvaluatorAlertButton/EvaluatorAlertButton";
type EvaluatorRow = RouterOutputs["evalsV2"]["list"]["evaluators"][number];
@@ -170,6 +172,10 @@ export default function EvaluatorsPage() {
const router = useRouter();
const capture = usePostHogClientCapture();
const projectId = router.query.projectId as string;
+ const evaluatorAlerts = useEvaluatorAlerts({
+ scope: "allEvaluators",
+ projectId,
+ });
const projectDefaultModel = useProjectDefaultModel({
projectId,
source: "overview",
@@ -617,10 +623,18 @@ export default function EvaluatorsPage() {
projectDefaultModel.connectionsPending ||
projectDefaultModel.update.isPending
}
+ borderVariant="contrast"
/>
</PopoverTrigger>
</JudgeModelPicker>
)}
+ {showOnboarding ? null : (
+ <EvaluatorAlertButton
+ scope="allEvaluators"
+ projectId={projectId}
+ {...evaluatorAlerts}
+ />
+ )}
<Button onClick={() => setGalleryOpen(true)}>
<Plus className="mr-2 h-4 w-4" />
New evaluator
diff --git a/web/src/features/filters/components/filter-builder.tsx b/web/src/features/filters/components/filter-builder.tsx
index b8b6388138..bf0d23a0c1 100644
--- a/web/src/features/filters/components/filter-builder.tsx
+++ b/web/src/features/filters/components/filter-builder.tsx
@@ -667,6 +667,10 @@ function FilterBuilderForm({
const stringObjectSuggest =
column?.type === "stringObject" &&
columnsWithCustomSelect.includes(column.id);
+ const stringSuggest =
+ column?.type === "string" &&
+ columnsWithCustomSelect.includes(column.id) &&
+ column.options !== undefined;
const columnLabel = column ? column.name : "Column";
const columnsForPicker = getColumnOptionsForFilterRow(
columns,
@@ -900,8 +904,11 @@ function FilterBuilderForm({
{
...filter,
operator: value as any,
- // Ensure null filters always have empty string value
- value: filter.type === "null" ? "" : (filter.value as any),
+ // Value-less operators keep an empty string for schema compatibility.
+ value:
+ filter.type === "null" || value === "is not empty"
+ ? ""
+ : (filter.value as any),
},
i,
);
@@ -925,7 +932,9 @@ function FilterBuilderForm({
const valueControl = keyPending ? (
<Input disabled />
- ) : stringObjectSuggest && filter.type === "stringObject" ? (
+ ) : filter.type === "string" &&
+ filter.operator === "is not empty" ? null : stringObjectSuggest &&
+ filter.type === "stringObject" ? (
<SingleSelect
title="Value"
className="min-w-[100px]"
@@ -935,6 +944,16 @@ function FilterBuilderForm({
disabled={disabled}
isCustomSelectEnabled
/>
+ ) : stringSuggest && filter.type === "string" ? (
+ <SingleSelect
+ title="Value"
+ className="min-w-[100px]"
+ options={column?.type === "string" ? (column.options ?? []) : []}
+ value={filter.value ?? undefined}
+ onValueChange={(value) => handleFilterChange({ ...filter, value }, i)}
+ disabled={disabled}
+ isCustomSelectEnabled
+ />
) : filter.type === "string" || filter.type === "stringObject" ? (
<Input
disabled={disabled}
diff --git a/web/src/features/filters/components/single-select.clienttest.tsx b/web/src/features/filters/components/single-select.clienttest.tsx
index ad2e6854d6..dc1c8ae86a 100644
--- a/web/src/features/filters/components/single-select.clienttest.tsx
+++ b/web/src/features/filters/components/single-select.clienttest.tsx
@@ -54,4 +54,23 @@ describe("SingleSelect", () => {
expect(onValueChange).toHaveBeenCalledWith("release");
});
+
+ it("renders display labels but commits stable option values", () => {
+ const onValueChange = vi.fn();
+ render(
+ <SingleSelect
+ title="Evaluator"
+ options={[
+ { value: "evaluator-1", displayValue: "Answer quality" },
+ { value: "evaluator-2", displayValue: "Hallucination" },
+ ]}
+ onValueChange={onValueChange}
+ />,
+ );
+
+ open();
+ fireEvent.click(screen.getByRole("option", { name: "Answer quality" }));
+
+ expect(onValueChange).toHaveBeenCalledWith("evaluator-1");
+ });
});
diff --git a/web/src/features/filters/config/monitors-config.ts b/web/src/features/filters/config/monitors-config.ts
index 845377065f..0fcdf477e3 100644
--- a/web/src/features/filters/config/monitors-config.ts
+++ b/web/src/features/filters/config/monitors-config.ts
@@ -1,26 +1,45 @@
import { monitorsTableCols } from "@langfuse/shared";
import type { FilterConfig } from "@/src/features/filters/lib/filter-config";
+const baseFacets: FilterConfig["facets"] = [
+ {
+ type: "categorical" as const,
+ column: "severity",
+ label: "Severity",
+ disableTextFilter: true,
+ },
+ {
+ type: "categorical" as const,
+ column: "tags",
+ label: "Tags",
+ },
+];
+
export const monitorFilterConfig: FilterConfig = {
tableName: "monitors",
-
columnDefinitions: monitorsTableCols,
-
defaultExpanded: ["severity", "tags"],
-
defaultSidebarCollapsed: false,
-
- facets: [
- {
- type: "categorical" as const,
- column: "severity",
- label: "Severity",
- disableTextFilter: true,
- },
- {
- type: "categorical" as const,
- column: "tags",
- label: "Tags",
- },
- ],
+ facets: baseFacets,
};
+
+/** Adds the evaluator facet only when the project has evaluator-specific alerts. */
+export const getMonitorFilterConfig = (
+ hasEvaluatorOptions: boolean,
+): FilterConfig =>
+ hasEvaluatorOptions
+ ? {
+ ...monitorFilterConfig,
+ defaultExpanded: ["severity", "evaluatorId", "tags"],
+ facets: [
+ baseFacets[0],
+ {
+ type: "categorical" as const,
+ column: "evaluatorId",
+ label: "Evaluator",
+ disableTextFilter: true,
+ },
+ baseFacets[1],
+ ],
+ }
+ : monitorFilterConfig;
diff --git a/web/src/features/metrics/components/MetricsFilterBuilder.clienttest.ts b/web/src/features/metrics/components/MetricsFilterBuilder.clienttest.ts
index 88cbcdc9a5..516821c6df 100644
--- a/web/src/features/metrics/components/MetricsFilterBuilder.clienttest.ts
+++ b/web/src/features/metrics/components/MetricsFilterBuilder.clienttest.ts
@@ -2,7 +2,7 @@
import { describe, expect, it } from "vitest";
-import { type FilterState } from "@langfuse/shared";
+import { filterOperators, type FilterState } from "@langfuse/shared";
import {
getMetricsColumnsWithCustomSelect,
@@ -54,6 +54,44 @@ describe("editorFiltersToViewFilters", () => {
),
).toEqual(filters);
});
+
+ it("canonicalizes current and legacy evaluator display labels", () => {
+ const canonical: FilterState = [
+ {
+ column: "evaluatorId",
+ type: "string",
+ operator: "=",
+ value: "evaluator-1",
+ },
+ ];
+ for (const column of ["Evaluator", "Evaluator ID"]) {
+ expect(
+ editorFiltersToViewFilters("observations", [
+ {
+ ...canonical[0],
+ column,
+ },
+ ]),
+ ).toEqual(canonical);
+ }
+ expect(
+ editorFiltersToViewFilters("observations", [
+ {
+ column: "Evaluator execution",
+ type: "boolean",
+ operator: "=",
+ value: false,
+ },
+ ]),
+ ).toEqual([
+ {
+ column: "isEvaluatorTest",
+ type: "boolean",
+ operator: "=",
+ value: false,
+ },
+ ]);
+ });
});
describe("resolvesToColumn", () => {
@@ -91,6 +129,23 @@ describe("resolvesToColumn", () => {
expect(resolves("isRootObservation", "observations", "v1")).toBe(false);
});
+ it("offers an is-not-empty operator for evaluator IDs", () => {
+ const evaluatorColumn = columnsFor("observations").find(
+ (column) => column.id === "evaluatorId",
+ );
+
+ expect(evaluatorColumn?.type).toBe("string");
+ expect(filterOperators.string).toContain("is not empty");
+ });
+
+ it("renders evaluator alert prefill filters", () => {
+ expect(resolves("evaluatorId", "observations", "v1")).toBe(true);
+ expect(resolves("evaluatorId", "observations", "v2")).toBe(true);
+ expect(resolves("isEvaluatorTest", "observations", "v2")).toBe(true);
+ expect(resolves("evaluatorId", "scores-categorical", "v2")).toBe(true);
+ expect(resolves("isEvaluatorTest", "scores-categorical", "v2")).toBe(true);
+ });
+
it("unmapped column: does not resolve", () => {
expect(resolves("totallyUnknownColumn", "observations")).toBe(false);
});
@@ -216,6 +271,27 @@ describe("buildV2FilterColumnsParams", () => {
expect(getMetricsColumnsWithCustomSelect(params)).toContain("experimentId");
});
+ it("labels evaluator ID suggestions by name while retaining ID values", () => {
+ const params = buildV2FilterColumnsParams({
+ view: "observations",
+ filterOptions: undefined,
+ datasets: undefined,
+ evaluatorOptions: [
+ { value: "evaluator-1", displayValue: "Answer quality" },
+ ],
+ });
+ const column = getMetricsFilterColumns(params).find(
+ (candidate) => candidate.id === "evaluatorId",
+ );
+
+ expect(column?.name).toBe("Evaluator");
+ expect(column?.type).toBe("string");
+ expect(column && "options" in column ? column.options : []).toEqual([
+ { value: "evaluator-1", displayValue: "Answer quality" },
+ ]);
+ expect(getMetricsColumnsWithCustomSelect(params)).toContain("evaluatorId");
+ });
+
it("wires metadata key suggestions into the Metadata column", () => {
const params = buildV2FilterColumnsParams({
view: "observations",
diff --git a/web/src/features/metrics/components/MetricsFilterBuilder.tsx b/web/src/features/metrics/components/MetricsFilterBuilder.tsx
index 698499862a..af0986a347 100644
--- a/web/src/features/metrics/components/MetricsFilterBuilder.tsx
+++ b/web/src/features/metrics/components/MetricsFilterBuilder.tsx
@@ -91,8 +91,30 @@ export const MetricsFilterBuilder = ({
version,
...props
}: MetricsFilterFetcherProps & { version: ViewVersion }) => {
- if (version === "v1") return <MetricsFilterBuilderV1 {...props} />;
- return <MetricsFilterBuilderV2 {...props} />;
+ const evaluatorOptions = api.evalsV2.options.useQuery(
+ { projectId: props.projectId, limit: 100 },
+ v2FilterOptionsQueryConfig,
+ );
+ const evaluatorNameOptions =
+ evaluatorOptions.data?.map(({ id, name }) => ({
+ value: id,
+ displayValue: name,
+ })) ?? [];
+
+ if (version === "v1") {
+ return (
+ <MetricsFilterBuilderV1
+ {...props}
+ evaluatorOptions={evaluatorNameOptions}
+ />
+ );
+ }
+ return (
+ <MetricsFilterBuilderV2
+ {...props}
+ evaluatorOptions={evaluatorNameOptions}
+ />
+ );
};
/** MetricsFilterDateRange is the preview/lookback window used to scope filter-value discovery. */
@@ -107,6 +129,11 @@ type MetricsFilterFetcherProps = {
onChange: (filters: FilterState) => void;
};
+type MetricsFilterFetcherWithEvaluatorOptionsProps =
+ MetricsFilterFetcherProps & {
+ evaluatorOptions: SingleValueOption[];
+ };
+
/** MetricsFilterBuilderV1 loads the v1 (traces + generations + project) filter options and renders the filter view. */
const MetricsFilterBuilderV1 = ({
view,
@@ -114,7 +141,8 @@ const MetricsFilterBuilderV1 = ({
dateRange,
filters,
onChange,
-}: MetricsFilterFetcherProps) => {
+ evaluatorOptions,
+}: MetricsFilterFetcherWithEvaluatorOptionsProps) => {
const traceFilterOptions = api.traces.filterOptions.useQuery(
{
projectId,
@@ -143,6 +171,7 @@ const MetricsFilterBuilderV1 = ({
traceFilterOptions: traceFilterOptions.data,
generationsFilterOptions: generationsFilterOptions.data,
environmentFilterOptions: environmentFilterOptions.data,
+ evaluatorOptions,
});
return (
@@ -163,7 +192,8 @@ const MetricsFilterBuilderV2 = ({
dateRange,
filters,
onChange,
-}: MetricsFilterFetcherProps) => {
+ evaluatorOptions,
+}: MetricsFilterFetcherWithEvaluatorOptionsProps) => {
const startTimeFilter = metricsFilterTimeFilter("startTime", dateRange);
const eventsFilterOptions = api.events.filterOptions.useQuery(
@@ -187,6 +217,7 @@ const MetricsFilterBuilderV2 = ({
filterOptions: eventsFilterOptions.data,
slowFilterOptions: slowEventsFilterOptions.data,
datasets: datasets.data,
+ evaluatorOptions,
metadataKeys: eventsFilterOptions.data?.metadataKeys?.map(
(row) => row.value,
),
@@ -286,6 +317,7 @@ const buildV1FilterColumnsParams = ({
traceFilterOptions,
generationsFilterOptions,
environmentFilterOptions,
+ evaluatorOptions = [],
}: {
view: MetricsFilterFetcherProps["view"];
traceFilterOptions: RouterOutputs["traces"]["filterOptions"] | undefined;
@@ -295,6 +327,7 @@ const buildV1FilterColumnsParams = ({
environmentFilterOptions:
| RouterOutputs["projects"]["environmentFilterOptions"]
| undefined;
+ evaluatorOptions?: SingleValueOption[];
}): GetMetricsFilterColumnsParams => ({
selectedView: view,
viewVersion: "v1",
@@ -320,6 +353,7 @@ const buildV1FilterColumnsParams = ({
releaseOptions: [],
scoreNameOptions: [],
experimentIdOptions: [],
+ evaluatorOptions,
metadataKeyOptions: [],
});
@@ -329,12 +363,14 @@ const buildV2FilterColumnsParams = ({
filterOptions,
slowFilterOptions,
datasets,
+ evaluatorOptions = [],
metadataKeys,
}: {
view: z.infer<typeof views>;
filterOptions: RouterOutputs["events"]["filterOptions"] | undefined;
slowFilterOptions?: RouterOutputs["events"]["filterOptions"];
datasets: Array<{ id: string; name: string }> | undefined;
+ evaluatorOptions?: SingleValueOption[];
metadataKeys?: string[];
}): GetMetricsFilterColumnsParams => {
const datasetIds = new Set(
@@ -365,6 +401,7 @@ const buildV2FilterColumnsParams = ({
experimentIdOptions: normalizeSingleValueOptions(
filterOptions?.experimentId,
),
+ evaluatorOptions,
metadataKeyOptions: metadataKeys ?? [],
};
};
diff --git a/web/src/features/metrics/metricsFilterColumns.ts b/web/src/features/metrics/metricsFilterColumns.ts
index 93fa508a03..b9a6d4ad0e 100644
--- a/web/src/features/metrics/metricsFilterColumns.ts
+++ b/web/src/features/metrics/metricsFilterColumns.ts
@@ -27,6 +27,7 @@ export type GetMetricsFilterColumnsParams = {
releaseOptions: SingleValueOption[];
scoreNameOptions: SingleValueOption[];
experimentIdOptions: SingleValueOption[];
+ evaluatorOptions: SingleValueOption[];
metadataKeyOptions: string[];
};
@@ -55,6 +56,7 @@ const getMetricsFilterColumnSpecs = ({
releaseOptions,
scoreNameOptions,
experimentIdOptions,
+ evaluatorOptions,
metadataKeyOptions,
}: GetMetricsFilterColumnsParams): MetricsFilterColumnSpec[] => {
// Value suggestions come from the v2 events filter-options; v1 keeps manual
@@ -158,16 +160,36 @@ const getMetricsFilterColumnSpecs = ({
// "Observation Name" on the observations view filters observations.name
// (mapped through dashboardUiTableToViewMapping). "Score Name" is omitted
// here because observations:scores is 1:n -- see LFE-9773.
- filterColumns.push({
- column: {
- name: "Observation Name",
- id: "observationName",
- type: "stringOptions",
- options: observationNameOptions,
- internal: "internalValue",
+ filterColumns.push(
+ {
+ column: {
+ name: "Observation Name",
+ id: "observationName",
+ type: "stringOptions",
+ options: observationNameOptions,
+ internal: "internalValue",
+ },
+ customSelect: true,
},
- customSelect: true,
- });
+ {
+ column: {
+ name: "Evaluator",
+ id: "evaluatorId",
+ type: "string",
+ internal: "internalValue",
+ options: evaluatorOptions,
+ },
+ customSelect: true,
+ },
+ {
+ column: {
+ name: "Evaluator test run",
+ id: "isEvaluatorTest",
+ type: "boolean",
+ internal: "internalValue",
+ },
+ },
+ );
}
if (
@@ -187,6 +209,24 @@ const getMetricsFilterColumnSpecs = ({
},
customSelect: true,
},
+ {
+ column: {
+ name: "Evaluator",
+ id: "evaluatorId",
+ type: "string",
+ internal: "internalValue",
+ options: evaluatorOptions,
+ },
+ customSelect: true,
+ },
+ {
+ column: {
+ name: "Evaluator test run",
+ id: "isEvaluatorTest",
+ type: "boolean",
+ internal: "internalValue",
+ },
+ },
);
}
diff --git a/web/src/features/monitors/components/DeleteMonitorButton.clienttest.tsx b/web/src/features/monitors/components/DeleteMonitorButton.clienttest.tsx
new file mode 100644
index 0000000000..ca19801d30
--- /dev/null
+++ b/web/src/features/monitors/components/DeleteMonitorButton.clienttest.tsx
@@ -0,0 +1,62 @@
+import {
+ fireEvent,
+ render,
+ screen,
+ waitFor,
+ within,
+} from "@testing-library/react";
+
+import { invalidateMonitorQueriesAfterDelete } from "@/src/features/monitors/fns/invalidateMonitorQueriesAfterDelete";
+import { DeleteMonitorButton } from "./DeleteMonitorButton";
+
+describe("DeleteMonitorButton", () => {
+ it("invalidates monitor lists without refetching the deleted monitor", async () => {
+ const monitors = {
+ all: { invalidate: vi.fn().mockResolvedValue(undefined) },
+ getFilterOptions: { invalidate: vi.fn().mockResolvedValue(undefined) },
+ linkedEvaluatorAlerts: {
+ invalidate: vi.fn().mockResolvedValue(undefined),
+ },
+ get: { invalidate: vi.fn().mockResolvedValue(undefined) },
+ invalidate: vi.fn().mockResolvedValue(undefined),
+ };
+
+ await invalidateMonitorQueriesAfterDelete(monitors);
+
+ expect(monitors.all.invalidate).toHaveBeenCalledOnce();
+ expect(monitors.getFilterOptions.invalidate).toHaveBeenCalledOnce();
+ expect(monitors.linkedEvaluatorAlerts.invalidate).toHaveBeenCalledOnce();
+ expect(monitors.get.invalidate).not.toHaveBeenCalled();
+ expect(monitors.invalidate).not.toHaveBeenCalled();
+ });
+
+ it("confirms before deleting an alert", async () => {
+ const onDelete = vi.fn().mockResolvedValue(undefined);
+ render(
+ <DeleteMonitorButton
+ monitorName="Evaluator cost alert"
+ deleting={false}
+ onDelete={onDelete}
+ />,
+ );
+
+ const trigger = screen.getByRole("button", { name: "Delete alert" });
+ expect(trigger.querySelector(".text-destructive")).toBeInTheDocument();
+
+ fireEvent.click(trigger);
+
+ const dialog = screen.getByRole("dialog");
+ expect(
+ within(dialog).getByRole("heading", { name: "Delete alert" }),
+ ).toBeInTheDocument();
+ expect(
+ within(dialog).getByText(/permanently deletes "Evaluator cost alert"/),
+ ).toBeInTheDocument();
+
+ fireEvent.click(
+ within(dialog).getByRole("button", { name: "Delete alert" }),
+ );
+
+ await waitFor(() => expect(onDelete).toHaveBeenCalledOnce());
+ });
+});
diff --git a/web/src/features/monitors/components/DeleteMonitorButton.tsx b/web/src/features/monitors/components/DeleteMonitorButton.tsx
new file mode 100644
index 0000000000..97c929cbed
--- /dev/null
+++ b/web/src/features/monitors/components/DeleteMonitorButton.tsx
@@ -0,0 +1,50 @@
+import { useState } from "react";
+import { Trash2 } from "lucide-react";
+
+import { Button } from "@/src/components/ui/button";
+import { ConfirmDialog } from "@/src/components/ui/confirm-dialog";
+
+export function DeleteMonitorButton({
+ monitorName,
+ deleting,
+ onDelete,
+}: {
+ monitorName: string;
+ deleting: boolean;
+ onDelete: () => Promise<void>;
+}) {
+ const [open, setOpen] = useState(false);
+
+ const handleDelete = async () => {
+ try {
+ await onDelete();
+ setOpen(false);
+ } catch {
+ // The mutation reports the error while the dialog remains open for retry.
+ }
+ };
+
+ return (
+ <ConfirmDialog
+ open={open}
+ onOpenChange={setOpen}
+ title="Delete alert"
+ description={`This permanently deletes "${monitorName}". This action cannot be undone.`}
+ confirmLabel="Delete alert"
+ loading={deleting}
+ loadingText="Deleting alert..."
+ onConfirm={handleDelete}
+ trigger={
+ <Button
+ type="button"
+ variant="outline"
+ size="icon"
+ title="Delete alert"
+ aria-label="Delete alert"
+ >
+ <Trash2 className="text-destructive h-4 w-4" />
+ </Button>
+ }
+ />
+ );
+}
diff --git a/web/src/features/monitors/components/MonitorAutomationsPanel.toggle.clienttest.tsx b/web/src/features/monitors/components/MonitorAutomationsPanel.toggle.clienttest.tsx
index e77816113f..8d02164784 100644
--- a/web/src/features/monitors/components/MonitorAutomationsPanel.toggle.clienttest.tsx
+++ b/web/src/features/monitors/components/MonitorAutomationsPanel.toggle.clienttest.tsx
@@ -24,7 +24,7 @@ vi.mock("@/src/utils/api", () => ({
import { MonitorAutomationsPanel } from "./MonitorAutomationsPanel";
-describe("MonitorAutomationsPanel toggle immutability", () => {
+describe("MonitorAutomationsPanel selection", () => {
it("does not mutate the memoized set: toggling the same unselected row twice reports it selected both times", () => {
useQueryMock.mockReturnValue({ data: [automationRow], isPending: false });
const onTriggerIdsChange = vi.fn();
diff --git a/web/src/features/monitors/components/MonitorForm.clienttest.ts b/web/src/features/monitors/components/MonitorForm.clienttest.ts
index 6b388e80a6..ebdf13ad2f 100644
--- a/web/src/features/monitors/components/MonitorForm.clienttest.ts
+++ b/web/src/features/monitors/components/MonitorForm.clienttest.ts
@@ -17,6 +17,7 @@ const {
monitorToDefaults,
nameOrPlaceholder,
resolveViewChangePatch,
+ monitorCreateAnalyticsProperties,
} = __test;
describe("createDefaults", () => {
@@ -72,6 +73,28 @@ describe("createDefaults", () => {
const defaults = createDefaults("project-1");
expect(defaults.triggerIds).toEqual([]);
});
+
+ it("uses a prefilled evaluator score window", () => {
+ expect(createDefaults("project-1", { window: "1d" }).window).toBe("1d");
+ });
+});
+
+describe("monitorCreateAnalyticsProperties", () => {
+ it("tracks the selected score window", () => {
+ expect(
+ monitorCreateAnalyticsProperties("evaluator_score", {
+ view: "scores-numeric",
+ metric: { measure: "value", aggregation: "avg" },
+ window: "1d",
+ }),
+ ).toEqual({
+ source: "evaluator_score",
+ view: "scores-numeric",
+ measure: "value",
+ aggregation: "avg",
+ window: "1d",
+ });
+ });
});
describe("nameOrPlaceholder", () => {
diff --git a/web/src/features/monitors/components/MonitorForm.tsx b/web/src/features/monitors/components/MonitorForm.tsx
index 46453e3cae..06a69801cf 100644
--- a/web/src/features/monitors/components/MonitorForm.tsx
+++ b/web/src/features/monitors/components/MonitorForm.tsx
@@ -7,6 +7,7 @@ import { zodResolver } from "@hookform/resolvers/zod";
import { startCase } from "lodash";
import { api } from "@/src/utils/api";
+import { AIAssistedInput } from "@/src/components/ui/ai-assisted-input";
import { Button } from "@/src/components/ui/button";
import {
Accordion,
@@ -41,11 +42,15 @@ import {
SelectValue,
} from "@/src/components/ui/select";
import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
+import { useLangfuseCloudRegion } from "@/src/features/organizations";
+import { useProject } from "@/src/features/projects";
import { showSuccessToast } from "@/src/features/notifications/showSuccessToast";
import { showErrorToast } from "@/src/features/notifications/showErrorToast";
import { WidgetPropertySelectItem } from "@/src/features/widgets/components/WidgetPropertySelectItem";
import { MetricsFilterBuilder } from "@/src/features/metrics/components/MetricsFilterBuilder";
import { partitionWidgetUiTableFiltersToView } from "@/src/features/dashboard/lib/dashboardUiTableToViewMapping";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics";
+import { resolveMonitorNameForSave } from "@/src/features/monitors/fns/resolveMonitorNameForSave";
import { cn } from "@/src/utils/tailwind";
import {
@@ -88,18 +93,24 @@ import {
} from "../helpers/renderMonitorLabels";
/** createDefaults returns the form defaults for a brand-new monitor. */
-const createDefaults = (projectId: string): Partial<CreateMonitor> => ({
+const createDefaults = (
+ projectId: string,
+ prefill?: Partial<
+ Pick<CreateMonitor, "view" | "filters" | "metric" | "window" | "tags">
+ >,
+ initialTriggerIds: string[] = [],
+): Partial<CreateMonitor> => ({
projectId,
- view: "observations",
- filters: [],
- metric: { measure: "count", aggregation: "count" },
- window: "5m",
+ view: prefill?.view ?? "observations",
+ filters: prefill?.filters ?? [],
+ metric: prefill?.metric ?? { measure: "count", aggregation: "count" },
+ window: prefill?.window ?? "5m",
thresholdOperator: MonitorThresholdOperatorSchema.enum.GT,
warningThreshold: null,
noData: { mode: MonitorNoDataModeSchema.enum.SUBSTITUTE_ZERO },
renotify: { mode: "OFF" },
- tags: [],
- triggerIds: [],
+ tags: prefill?.tags ?? [],
+ triggerIds: initialTriggerIds,
status: MonitorStatusSchema.enum.ACTIVE,
});
@@ -140,19 +151,49 @@ const nameOrPlaceholder = (
placeholder: string,
): string => name || placeholder;
+type MonitorAnalyticsSource =
+ | "alerts"
+ | "evaluator_score"
+ | "evaluator_cost"
+ | "all_evaluator_cost";
+
+const monitorCreateAnalyticsProperties = (
+ source: MonitorAnalyticsSource,
+ monitor: Pick<CreateMonitor, "view" | "metric" | "window">,
+) => ({
+ source,
+ view: monitor.view,
+ measure: monitor.metric.measure,
+ aggregation: monitor.metric.aggregation,
+ window: monitor.window,
+});
+
/** MonitorForm renders the create/edit form for a Monitor. */
export const MonitorForm = ({
projectId,
monitor,
+ prefill,
+ analyticsSource = "alerts",
+ initialTriggerIds = [],
onNameChange,
}: {
projectId: string;
monitor?: Monitor;
+ prefill?: Partial<
+ Pick<CreateMonitor, "view" | "filters" | "metric" | "window" | "tags">
+ >;
+ analyticsSource?: MonitorAnalyticsSource;
+ initialTriggerIds?: string[];
/** onNameChange fires on every form change so the host (e.g. the edit page header) can mirror the live name. */
onNameChange?: (name: string) => void;
}) => {
/** router is the Next router used to redirect after a successful create. */
const router = useRouter();
+ const capture = usePostHogClientCapture();
+ const { isLangfuseCloud } = useLangfuseCloudRegion();
+ const { organization } = useProject(projectId);
+ const nameAIAssistanceAvailable =
+ isLangfuseCloud && Boolean(organization?.aiFeaturesEnabled);
/** isEdit is true when the form is bound to an existing monitor. */
const isEdit = Boolean(monitor);
/** hasAccess gates write controls behind the alerts:CUD RBAC scope. */
@@ -165,7 +206,7 @@ export const MonitorForm = ({
/** defaultValues seeds the form from the existing monitor on edit, otherwise from createDefaults. */
const defaultValues = isEdit
? monitorToDefaults(monitor as Monitor)
- : createDefaults(projectId);
+ : createDefaults(projectId, prefill, initialTriggerIds);
/** namePlaceholderRef holds the latest computed name placeholder for the resolver. */
const namePlaceholderRef = useRef("");
@@ -190,10 +231,42 @@ export const MonitorForm = ({
mode: "onChange",
});
+ const suggestName = api.monitors.suggestName.useMutation();
+ const generateNameSuggestion = async (): Promise<string | null> => {
+ if (!nameAIAssistanceAvailable) return null;
+ try {
+ return await suggestName.mutateAsync({
+ projectId,
+ description: namePlaceholderRef.current,
+ });
+ } catch {
+ return null;
+ }
+ };
+ const requestNameSuggestion = async () => {
+ const generatedName = await generateNameSuggestion();
+ if (!generatedName) {
+ showErrorToast(
+ "Couldn't generate an alert title",
+ "Please enter a title manually.",
+ );
+ return;
+ }
+ form.setValue("name", generatedName, {
+ shouldDirty: true,
+ shouldValidate: true,
+ });
+ onNameChange?.(generatedName);
+ };
+
/** createMutation creates a new monitor and returns to the monitors list on success. */
const createMutation = api.monitors.create.useMutation({
onSuccess: async (_data, variables) => {
await utils.monitors.invalidate();
+ capture(
+ "monitors:create",
+ monitorCreateAnalyticsProperties(analyticsSource, variables),
+ );
showSuccessToast({
title: "Alert created",
description: `"${variables.name}" is now active.`,
@@ -218,9 +291,24 @@ export const MonitorForm = ({
/** onSubmit strips unsupported filter rows and dispatches the create or update mutation. */
const onSubmit = form.handleSubmit(
- (values) => {
+ async (values) => {
+ const resolvedName = await resolveMonitorNameForSave({
+ name: form.getValues("name"),
+ fallbackName: namePlaceholderRef.current,
+ aiAvailable: nameAIAssistanceAvailable,
+ generateName: generateNameSuggestion,
+ });
+ if (!resolvedName) {
+ showErrorToast(
+ "Couldn't generate an alert title",
+ "Please enter a title manually and try again.",
+ );
+ return;
+ }
+
const normalizedValues = {
...values,
+ name: resolvedName,
filters: partitionWidgetUiTableFiltersToView(
values.view as Parameters<
typeof partitionWidgetUiTableFiltersToView
@@ -327,7 +415,8 @@ export const MonitorForm = ({
const submitting =
form.formState.isSubmitting ||
createMutation.isPending ||
- updateMutation.isPending;
+ updateMutation.isPending ||
+ suggestName.isPending;
return (
<Form {...form}>
@@ -709,18 +798,33 @@ export const MonitorForm = ({
name="name"
render={({ field }) => (
<FormItem>
- <FormLabel>Name</FormLabel>
+ <FormLabel>Title</FormLabel>
<FormControl>
- <Input
+ <AIAssistedInput
+ id="monitor-title"
maxLength={200}
placeholder={namePlaceholder}
disabled={!hasAccess}
- {...field}
value={field.value ?? ""}
onChange={(e) => {
field.onChange(e);
onNameChange?.(e.target.value ?? "");
}}
+ fieldName="title"
+ aiAssistance={
+ !nameAIAssistanceAvailable
+ ? { state: "unavailable" }
+ : suggestName.isPending
+ ? { state: "generating" }
+ : {
+ state: "idle",
+ onGenerate: () => {
+ requestNameSuggestion().catch(
+ () => undefined,
+ );
+ },
+ }
+ }
/>
</FormControl>
<FormMessage />
@@ -1017,4 +1121,5 @@ export const __test = {
monitorToDefaults,
nameOrPlaceholder,
resolveViewChangePatch,
+ monitorCreateAnalyticsProperties,
};
diff --git a/web/src/features/monitors/components/MonitorSeverityBadge.tsx b/web/src/features/monitors/components/MonitorSeverityBadge.tsx
index a1b93ae6a8..b28fbc32a7 100644
--- a/web/src/features/monitors/components/MonitorSeverityBadge.tsx
+++ b/web/src/features/monitors/components/MonitorSeverityBadge.tsx
@@ -1,6 +1,4 @@
/* eslint-disable @repo/no-style-props */
-import { PauseCircle } from "lucide-react";
-
import { Badge, type BadgeProps } from "@/src/components/ui/badge";
import { cn } from "@/src/utils/tailwind";
import { type MonitorSeverity } from "@langfuse/shared/monitors";
@@ -9,7 +7,7 @@ import { type MonitorSeverity } from "@langfuse/shared/monitors";
const severityVariant: Record<MonitorSeverity, BadgeProps["variant"]> = {
UNKNOWN: "outline",
NO_DATA: "secondary",
- PAUSED: "outline",
+ PAUSED: "default",
OK: "default",
WARNING: "default",
ALERT: "default",
@@ -19,7 +17,7 @@ const severityVariant: Record<MonitorSeverity, BadgeProps["variant"]> = {
const severityClassName: Record<MonitorSeverity, string> = {
UNKNOWN: "text-gray-400",
NO_DATA: "",
- PAUSED: "text-gray-400",
+ PAUSED: "bg-gray-500 text-white hover:bg-gray-500",
OK: "bg-emerald-500 text-white hover:bg-emerald-500",
WARNING: "bg-amber-600 text-white hover:bg-amber-600",
ALERT: "bg-orange-600 text-white hover:bg-orange-600",
@@ -52,12 +50,7 @@ export function MonitorSeverityBadge({
className,
)}
>
- {severity === "PAUSED" ? (
- <span className="inline-flex items-center gap-1">
- <PauseCircle className="h-4 w-4" />
- {severityLabel.PAUSED}
- </span>
- ) : severity === "UNKNOWN" ? (
+ {severity === "UNKNOWN" ? (
<span className="inline-flex items-center gap-1.5">
<span
className="h-3.5 w-3.5 animate-spin rounded-full border-2 border-gray-200 border-t-gray-400"
diff --git a/web/src/features/monitors/components/MonitorsTable.clienttest.ts b/web/src/features/monitors/components/MonitorsTable.clienttest.ts
index d9ab283efd..631c02e56e 100644
--- a/web/src/features/monitors/components/MonitorsTable.clienttest.ts
+++ b/web/src/features/monitors/components/MonitorsTable.clienttest.ts
@@ -9,6 +9,7 @@ import {
MonitorThresholdOperatorSchema,
} from "@langfuse/shared/monitors";
+import { getMonitorFilterConfig } from "@/src/features/filters/config/monitors-config";
import { __test } from "./MonitorsTable";
const { filterStateToListMonitorFilter, buildStatusToggleUpdate } = __test;
@@ -43,6 +44,21 @@ const monitorFixture = (status: Monitor["status"]): Monitor => ({
lastCompletedAt: null,
});
+describe("getMonitorFilterConfig", () => {
+ it("shows the evaluator facet only when evaluator-specific alerts exist", () => {
+ expect(
+ getMonitorFilterConfig(false).facets.some(
+ (facet) => facet.column === "evaluatorId",
+ ),
+ ).toBe(false);
+ expect(
+ getMonitorFilterConfig(true).facets.some(
+ (facet) => facet.column === "evaluatorId",
+ ),
+ ).toBe(true);
+ });
+});
+
describe("filterStateToListMonitorFilter", () => {
it("passes severity `any of` through with values intact", () => {
const state: FilterState = [
@@ -140,6 +156,18 @@ describe("filterStateToListMonitorFilter", () => {
]);
});
+ it("passes evaluator ID rows through unchanged", () => {
+ const state: FilterState = [
+ {
+ type: "stringOptions",
+ column: "evaluatorId",
+ operator: "any of",
+ value: ["eval-1"],
+ },
+ ];
+ expect(filterStateToListMonitorFilter(state)).toEqual(state);
+ });
+
it("passes tags rows through unchanged", () => {
const state: FilterState = [
{
diff --git a/web/src/features/monitors/components/MonitorsTable.tsx b/web/src/features/monitors/components/MonitorsTable.tsx
index ba2a109894..f6e6fefea8 100644
--- a/web/src/features/monitors/components/MonitorsTable.tsx
+++ b/web/src/features/monitors/components/MonitorsTable.tsx
@@ -18,7 +18,7 @@ import {
DropdownMenuItem,
DropdownMenuTrigger,
} from "@/src/components/ui/dropdown-menu";
-import { monitorFilterConfig } from "@/src/features/filters/config/monitors-config";
+import { getMonitorFilterConfig } from "@/src/features/filters/config/monitors-config";
import { useSidebarFilterState } from "@/src/features/filters/hooks/useSidebarFilterState";
import { showErrorToast } from "@/src/features/notifications/showErrorToast";
import { showSuccessToast } from "@/src/features/notifications/showSuccessToast";
@@ -119,10 +119,17 @@ export function MonitorsTable() {
displayValue: value.replace(/_/g, " "),
})),
tags: filterOptions.data?.tags.map((t) => ({ value: t.value })) ?? [],
+ evaluatorId: filterOptions.data?.evaluators ?? [],
}),
[filterOptions.data],
);
+ const monitorFilterConfig = useMemo(
+ () =>
+ getMonitorFilterConfig((filterOptions.data?.evaluators.length ?? 0) > 0),
+ [filterOptions.data?.evaluators.length],
+ );
+
/** queryFilter is the bound sidebar filter state, synced to the URL and to session storage per project. */
const queryFilter = useSidebarFilterState(
monitorFilterConfig,
diff --git a/web/src/features/monitors/fns/getInitialMonitorTriggerIds.clienttest.ts b/web/src/features/monitors/fns/getInitialMonitorTriggerIds.clienttest.ts
new file mode 100644
index 0000000000..51a161519a
--- /dev/null
+++ b/web/src/features/monitors/fns/getInitialMonitorTriggerIds.clienttest.ts
@@ -0,0 +1,20 @@
+// @vitest-environment node
+
+import { describe, expect, it } from "vitest";
+
+import { getInitialMonitorTriggerIds } from "./getInitialMonitorTriggerIds";
+
+describe("getInitialMonitorTriggerIds", () => {
+ it("selects the first available automation", () => {
+ expect(
+ getInitialMonitorTriggerIds([
+ { trigger: { id: "trigger-1" } },
+ { trigger: { id: "trigger-2" } },
+ ] as Parameters<typeof getInitialMonitorTriggerIds>[0]),
+ ).toEqual(["trigger-1"]);
+ });
+
+ it("returns no selection without an available automation", () => {
+ expect(getInitialMonitorTriggerIds([])).toEqual([]);
+ });
+});
diff --git a/web/src/features/monitors/fns/getInitialMonitorTriggerIds.ts b/web/src/features/monitors/fns/getInitialMonitorTriggerIds.ts
new file mode 100644
index 0000000000..c840bccc67
--- /dev/null
+++ b/web/src/features/monitors/fns/getInitialMonitorTriggerIds.ts
@@ -0,0 +1,9 @@
+import type { AutomationDomain } from "@langfuse/shared";
+
+/** Selects the first available automation for a new monitor. */
+export function getInitialMonitorTriggerIds(
+ automations: AutomationDomain[],
+): string[] {
+ const firstTriggerId = automations[0]?.trigger.id;
+ return firstTriggerId ? [firstTriggerId] : [];
+}
diff --git a/web/src/features/monitors/fns/getMonitorPrefill.clienttest.ts b/web/src/features/monitors/fns/getMonitorPrefill.clienttest.ts
new file mode 100644
index 0000000000..11ce722529
--- /dev/null
+++ b/web/src/features/monitors/fns/getMonitorPrefill.clienttest.ts
@@ -0,0 +1,96 @@
+import { describe, expect, it } from "vitest";
+
+import { getMonitorPrefill } from "@/src/features/monitors/fns/getMonitorPrefill";
+
+const evaluatorFilters = [
+ {
+ column: "evaluatorId",
+ type: "string",
+ operator: "=",
+ value: "evaluator-1",
+ },
+ {
+ column: "isEvaluatorTest",
+ type: "boolean",
+ operator: "=",
+ value: false,
+ },
+];
+
+describe("getMonitorPrefill", () => {
+ it.each([
+ ["NUMERIC", "scores-numeric"],
+ ["BOOLEAN", "scores-boolean"],
+ ["CATEGORICAL", "scores-categorical"],
+ ] as const)("prefills a %s evaluator score alert", (scoreDataType, view) => {
+ expect(
+ getMonitorPrefill({
+ alert: "evaluatorScore",
+ evaluatorId: "evaluator-1",
+ scoreDataType,
+ }),
+ ).toMatchObject({
+ view,
+ filters: evaluatorFilters,
+ window: "1d",
+ tags: ["evaluators", "evaluator:evaluator-1"],
+ metric:
+ scoreDataType === "CATEGORICAL"
+ ? { measure: "count", aggregation: "count" }
+ : { measure: "value", aggregation: "avg" },
+ });
+ });
+
+ it("prefills evaluator cost without evaluator test runs", () => {
+ expect(
+ getMonitorPrefill({
+ alert: "evaluatorCost",
+ evaluatorId: "evaluator-1",
+ }),
+ ).toEqual({
+ view: "observations",
+ filters: evaluatorFilters,
+ metric: { measure: "totalCost", aggregation: "sum" },
+ tags: ["evaluators", "evaluator:evaluator-1"],
+ });
+ });
+
+ it("prefills aggregate evaluator cost across current and legacy executions", () => {
+ expect(getMonitorPrefill({ alert: "allEvaluatorCost" })).toEqual({
+ view: "observations",
+ filters: [
+ {
+ column: "evaluatorId",
+ type: "string",
+ operator: "is not empty",
+ value: "",
+ },
+ {
+ column: "isEvaluatorTest",
+ type: "boolean",
+ operator: "=",
+ value: false,
+ },
+ ],
+ metric: { measure: "totalCost", aggregation: "sum" },
+ tags: ["evaluators"],
+ });
+ });
+
+ it.each([
+ {},
+ { alert: "unknown" },
+ { alert: "evaluatorCost" },
+ {
+ alert: "evaluatorScore",
+ evaluatorId: "evaluator-1",
+ scoreDataType: "TEXT",
+ },
+ {
+ alert: ["evaluatorCost"],
+ evaluatorId: "evaluator-1",
+ },
+ ])("ignores invalid query values: %j", (query) => {
+ expect(getMonitorPrefill(query)).toBeUndefined();
+ });
+});
diff --git a/web/src/features/monitors/fns/getMonitorPrefill.ts b/web/src/features/monitors/fns/getMonitorPrefill.ts
new file mode 100644
index 0000000000..3210f034d8
--- /dev/null
+++ b/web/src/features/monitors/fns/getMonitorPrefill.ts
@@ -0,0 +1,93 @@
+import type { CreateMonitor } from "@langfuse/shared/monitors";
+
+const evaluatorIdFilter = (evaluatorId: string) => ({
+ column: "evaluatorId",
+ type: "string" as const,
+ operator: "=" as const,
+ value: evaluatorId,
+});
+
+const anyEvaluatorIdFilter = {
+ column: "evaluatorId",
+ type: "string" as const,
+ operator: "is not empty" as const,
+ value: "",
+};
+
+const excludeEvaluatorTestsFilter = {
+ column: "isEvaluatorTest",
+ type: "boolean" as const,
+ operator: "=" as const,
+ value: false,
+};
+
+const evaluatorAlertTags = ["evaluators"];
+const specificEvaluatorAlertTags = (evaluatorId: string) => [
+ ...evaluatorAlertTags,
+ `evaluator:${evaluatorId}`,
+];
+
+/** Returns the supported alert-form defaults encoded by the new-alert URL. */
+export function getMonitorPrefill(
+ query: Record<string, string | string[] | undefined>,
+): Partial<CreateMonitor> | undefined {
+ const alert = query.alert;
+
+ if (alert === "allEvaluatorCost") {
+ return {
+ view: "observations",
+ filters: [anyEvaluatorIdFilter, excludeEvaluatorTestsFilter],
+ metric: { measure: "totalCost", aggregation: "sum" },
+ tags: evaluatorAlertTags,
+ };
+ }
+
+ const evaluatorId = query.evaluatorId;
+ if (typeof evaluatorId !== "string" || evaluatorId.length === 0) {
+ return undefined;
+ }
+
+ if (alert === "evaluatorCost") {
+ return {
+ view: "observations",
+ filters: [evaluatorIdFilter(evaluatorId), excludeEvaluatorTestsFilter],
+ metric: { measure: "totalCost", aggregation: "sum" },
+ tags: specificEvaluatorAlertTags(evaluatorId),
+ };
+ }
+
+ if (alert !== "evaluatorScore") {
+ return undefined;
+ }
+
+ const filters = [evaluatorIdFilter(evaluatorId), excludeEvaluatorTestsFilter];
+
+ switch (query.scoreDataType) {
+ case "NUMERIC":
+ return {
+ view: "scores-numeric",
+ filters,
+ metric: { measure: "value", aggregation: "avg" },
+ window: "1d",
+ tags: specificEvaluatorAlertTags(evaluatorId),
+ };
+ case "BOOLEAN":
+ return {
+ view: "scores-boolean",
+ filters,
+ metric: { measure: "value", aggregation: "avg" },
+ window: "1d",
+ tags: specificEvaluatorAlertTags(evaluatorId),
+ };
+ case "CATEGORICAL":
+ return {
+ view: "scores-categorical",
+ filters,
+ metric: { measure: "count", aggregation: "count" },
+ window: "1d",
+ tags: specificEvaluatorAlertTags(evaluatorId),
+ };
+ default:
+ return undefined;
+ }
+}
diff --git a/web/src/features/monitors/fns/invalidateMonitorQueriesAfterDelete.ts b/web/src/features/monitors/fns/invalidateMonitorQueriesAfterDelete.ts
new file mode 100644
index 0000000000..e88a4d026a
--- /dev/null
+++ b/web/src/features/monitors/fns/invalidateMonitorQueriesAfterDelete.ts
@@ -0,0 +1,16 @@
+type MonitorQueryInvalidations = {
+ all: { invalidate: () => Promise<unknown> };
+ getFilterOptions: { invalidate: () => Promise<unknown> };
+ linkedEvaluatorAlerts: { invalidate: () => Promise<unknown> };
+};
+
+/** Refreshes monitor collections without refetching the deleted monitor detail. */
+export async function invalidateMonitorQueriesAfterDelete(
+ monitors: MonitorQueryInvalidations,
+): Promise<void> {
+ await Promise.all([
+ monitors.all.invalidate(),
+ monitors.getFilterOptions.invalidate(),
+ monitors.linkedEvaluatorAlerts.invalidate(),
+ ]);
+}
diff --git a/web/src/features/monitors/fns/resolveMonitorNameForSave.clienttest.ts b/web/src/features/monitors/fns/resolveMonitorNameForSave.clienttest.ts
new file mode 100644
index 0000000000..ab185bbd5a
--- /dev/null
+++ b/web/src/features/monitors/fns/resolveMonitorNameForSave.clienttest.ts
@@ -0,0 +1,59 @@
+// @vitest-environment node
+
+import { describe, expect, it, vi } from "vitest";
+
+import { resolveMonitorNameForSave } from "./resolveMonitorNameForSave";
+
+describe("resolveMonitorNameForSave", () => {
+ it("keeps a manually entered name", async () => {
+ const generateName = vi.fn();
+
+ await expect(
+ resolveMonitorNameForSave({
+ name: "Cost spike",
+ fallbackName: "Fallback",
+ aiAvailable: true,
+ generateName,
+ }),
+ ).resolves.toBe("Cost spike");
+ expect(generateName).not.toHaveBeenCalled();
+ });
+
+ it("generates an empty name when AI assistance is available", async () => {
+ const generateName = vi.fn().mockResolvedValue(" Evaluator cost spike ");
+
+ await expect(
+ resolveMonitorNameForSave({
+ name: "",
+ fallbackName: "Fallback",
+ aiAvailable: true,
+ generateName,
+ }),
+ ).resolves.toBe("Evaluator cost spike");
+ });
+
+ it("uses the deterministic fallback when AI assistance is unavailable", async () => {
+ const generateName = vi.fn();
+
+ await expect(
+ resolveMonitorNameForSave({
+ name: "",
+ fallbackName: "Count of observations is above 5",
+ aiAvailable: false,
+ generateName,
+ }),
+ ).resolves.toBe("Count of observations is above 5");
+ expect(generateName).not.toHaveBeenCalled();
+ });
+
+ it("returns null when generation fails", async () => {
+ await expect(
+ resolveMonitorNameForSave({
+ name: "",
+ fallbackName: "Fallback",
+ aiAvailable: true,
+ generateName: vi.fn().mockResolvedValue(null),
+ }),
+ ).resolves.toBeNull();
+ });
+});
diff --git a/web/src/features/monitors/fns/resolveMonitorNameForSave.ts b/web/src/features/monitors/fns/resolveMonitorNameForSave.ts
new file mode 100644
index 0000000000..588a45dfa3
--- /dev/null
+++ b/web/src/features/monitors/fns/resolveMonitorNameForSave.ts
@@ -0,0 +1,19 @@
+/** Resolves a monitor name at save time, generating blank names when AI assistance is available. */
+export async function resolveMonitorNameForSave({
+ name,
+ fallbackName,
+ aiAvailable,
+ generateName,
+}: {
+ name: string | undefined;
+ fallbackName: string;
+ aiAvailable: boolean;
+ generateName: () => Promise<string | null>;
+}): Promise<string | null> {
+ const enteredName = name?.trim();
+ if (enteredName) return enteredName;
+ if (!aiAvailable) return fallbackName;
+
+ const generatedName = (await generateName())?.trim();
+ return generatedName || null;
+}
diff --git a/web/src/features/monitors/index.ts b/web/src/features/monitors/index.ts
new file mode 100644
index 0000000000..9b22c7fc62
--- /dev/null
+++ b/web/src/features/monitors/index.ts
@@ -0,0 +1 @@
+export { MonitorSeverityBadge } from "@/src/features/monitors/components/MonitorSeverityBadge";
diff --git a/web/src/features/monitors/pages/EditMonitorPage.tsx b/web/src/features/monitors/pages/EditMonitorPage.tsx
index 1df94c7e00..c31af08b95 100644
--- a/web/src/features/monitors/pages/EditMonitorPage.tsx
+++ b/web/src/features/monitors/pages/EditMonitorPage.tsx
@@ -3,8 +3,13 @@ import { useState } from "react";
import { ErrorPage } from "@/src/components/error-page";
import Page from "@/src/components/layouts/page";
+import { DeleteMonitorButton } from "@/src/features/monitors/components/DeleteMonitorButton";
import { MonitorForm } from "@/src/features/monitors/components/MonitorForm";
import { MonitorPagePermissions } from "@/src/features/monitors/components/MonitorPagePermissions";
+import { invalidateMonitorQueriesAfterDelete } from "@/src/features/monitors/fns/invalidateMonitorQueriesAfterDelete";
+import { showErrorToast } from "@/src/features/notifications/showErrorToast";
+import { showSuccessToast } from "@/src/features/notifications/showSuccessToast";
+import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
import { api, type APIError } from "@/src/utils/api";
import { type Monitor } from "@langfuse/shared/monitors";
@@ -41,10 +46,44 @@ function EditMonitorPageRouter() {
/** EditMonitorFormPage renders the edit monitors form */
const EditMonitorFormPage = ({ monitor }: { monitor: Monitor }) => {
+ const router = useRouter();
+ const utils = api.useUtils();
const [liveName, setLiveName] = useState(monitor.name);
+ const canDelete = useHasProjectAccess({
+ projectId: monitor.projectId,
+ scope: "alerts:CUD",
+ });
+ const deleteMonitor = api.monitors.delete.useMutation({
+ onSuccess: async () => {
+ await invalidateMonitorQueriesAfterDelete(utils.monitors);
+ showSuccessToast({
+ title: "Alert deleted",
+ description: `"${monitor.name}" has been deleted.`,
+ });
+ await router.replace(`/project/${monitor.projectId}/alerts`);
+ },
+ onError: (error) => showErrorToast("Failed to delete alert", error.message),
+ });
return (
- <Page withPadding headerProps={getHeaderProps(monitor.projectId, liveName)}>
+ <Page
+ withPadding
+ headerProps={{
+ ...getHeaderProps(monitor.projectId, liveName),
+ actionButtonsRight: canDelete ? (
+ <DeleteMonitorButton
+ monitorName={monitor.name}
+ deleting={deleteMonitor.isPending}
+ onDelete={async () => {
+ await deleteMonitor.mutateAsync({
+ projectId: monitor.projectId,
+ id: monitor.id,
+ });
+ }}
+ />
+ ) : undefined,
+ }}
+ >
<MonitorForm
projectId={monitor.projectId}
monitor={monitor}
diff --git a/web/src/features/monitors/pages/NewMonitorPage.tsx b/web/src/features/monitors/pages/NewMonitorPage.tsx
index fb255ede59..57a20c2d4d 100644
--- a/web/src/features/monitors/pages/NewMonitorPage.tsx
+++ b/web/src/features/monitors/pages/NewMonitorPage.tsx
@@ -1,13 +1,42 @@
import { useRouter } from "next/router";
import Page from "@/src/components/layouts/page";
+import { Skeleton } from "@/src/components/ui/skeleton";
import { MonitorForm } from "@/src/features/monitors/components/MonitorForm";
import { MonitorPagePermissions } from "@/src/features/monitors/components/MonitorPagePermissions";
+import { getInitialMonitorTriggerIds } from "@/src/features/monitors/fns/getInitialMonitorTriggerIds";
+import { getMonitorPrefill } from "@/src/features/monitors/fns/getMonitorPrefill";
+import { api } from "@/src/utils/api";
+import { TriggerEventSource } from "@langfuse/shared";
+
+const alertAnalyticsSource = (alert: string | string[] | undefined) => {
+ switch (alert) {
+ case "evaluatorScore":
+ return "evaluator_score" as const;
+ case "evaluatorCost":
+ return "evaluator_cost" as const;
+ case "allEvaluatorCost":
+ return "all_evaluator_cost" as const;
+ default:
+ return "alerts" as const;
+ }
+};
/** NewMonitorPage renders the create-monitor form for a project. */
export default function NewMonitorPage() {
const router = useRouter();
const projectId = router.query.projectId as string;
+ const prefill = router.isReady ? getMonitorPrefill(router.query) : undefined;
+ const analyticsSource = alertAnalyticsSource(router.query.alert);
+ const automations = api.automations.getAutomations.useQuery(
+ { projectId, eventSource: TriggerEventSource.Monitor },
+ {
+ enabled: Boolean(router.isReady && projectId),
+ trpc: { context: { skipBatch: true } },
+ refetchOnWindowFocus: false,
+ },
+ );
+ const initialTriggerIds = getInitialMonitorTriggerIds(automations.data ?? []);
return (
<MonitorPagePermissions scope="alerts:CUD">
@@ -20,7 +49,16 @@ export default function NewMonitorPage() {
],
}}
>
- <MonitorForm projectId={projectId} />
+ {router.isReady && !automations.isPending ? (
+ <MonitorForm
+ projectId={projectId}
+ prefill={prefill}
+ analyticsSource={analyticsSource}
+ initialTriggerIds={initialTriggerIds}
+ />
+ ) : (
+ <Skeleton className="h-96 w-full" />
+ )}
</Page>
</MonitorPagePermissions>
);
diff --git a/web/src/features/monitors/server/index.ts b/web/src/features/monitors/server/index.ts
new file mode 100644
index 0000000000..6dee3a99a5
--- /dev/null
+++ b/web/src/features/monitors/server/index.ts
@@ -0,0 +1,2 @@
+export { listLinkedEvaluatorAlerts } from "@/src/features/monitors/server/listLinkedEvaluatorAlerts";
+export { suggestMonitorName } from "@/src/features/monitors/server/suggestMonitorName";
diff --git a/web/src/features/monitors/server/listLinkedEvaluatorAlerts.servertest.ts b/web/src/features/monitors/server/listLinkedEvaluatorAlerts.servertest.ts
new file mode 100644
index 0000000000..49091629f1
--- /dev/null
+++ b/web/src/features/monitors/server/listLinkedEvaluatorAlerts.servertest.ts
@@ -0,0 +1,115 @@
+import { describe, expect, it, vi } from "vitest";
+
+import { listLinkedEvaluatorAlerts } from "./listLinkedEvaluatorAlerts";
+
+const prismaWithFindMany = (rows: unknown[] = [], linkedIds: string[] = []) => {
+ const findMany = vi.fn().mockResolvedValue(rows);
+ const $queryRaw = vi.fn().mockResolvedValue(linkedIds.map((id) => ({ id })));
+ return {
+ findMany,
+ $queryRaw,
+ prisma: { monitor: { findMany }, $queryRaw } as unknown as Parameters<
+ typeof listLinkedEvaluatorAlerts
+ >[0],
+ };
+};
+
+describe("listLinkedEvaluatorAlerts", () => {
+ it("filters evaluator connections in the database", async () => {
+ const { prisma, findMany, $queryRaw } = prismaWithFindMany(
+ [],
+ ["monitor-1"],
+ );
+
+ await listLinkedEvaluatorAlerts(prisma, {
+ scope: "evaluator",
+ projectId: "project-1",
+ evaluatorId: "evaluator-1",
+ });
+
+ expect(findMany).toHaveBeenCalledWith(
+ expect.objectContaining({
+ where: {
+ projectId: "project-1",
+ id: { in: ["monitor-1"] },
+ },
+ }),
+ );
+ expect($queryRaw).toHaveBeenCalledOnce();
+ });
+
+ it("returns at most 20 alerts and reports when more exist", async () => {
+ const rows = Array.from({ length: 21 }, (_, index) => ({
+ id: `alert-${index}`,
+ name: `Alert ${index}`,
+ status: "ACTIVE",
+ severity: "UNKNOWN",
+ metric: { measure: "count", aggregation: "count" },
+ thresholdOperator: "GT",
+ alertThreshold: { toNumber: () => index },
+ alertedAt: null,
+ }));
+ const { prisma, findMany } = prismaWithFindMany(
+ rows,
+ rows.map((row) => row.id),
+ );
+
+ const result = await listLinkedEvaluatorAlerts(prisma, {
+ scope: "evaluator",
+ projectId: "project-1",
+ evaluatorId: "evaluator-1",
+ });
+
+ expect(findMany).toHaveBeenCalledWith(
+ expect.objectContaining({ take: 21 }),
+ );
+ expect(result.data).toHaveLength(20);
+ expect(result.hasMore).toBe(true);
+ });
+
+ it("filters aggregate evaluator cost alerts in the database", async () => {
+ const { prisma, findMany } = prismaWithFindMany();
+
+ await listLinkedEvaluatorAlerts(prisma, {
+ scope: "allEvaluators",
+ projectId: "project-1",
+ });
+
+ expect(findMany).toHaveBeenCalledWith(
+ expect.objectContaining({
+ where: {
+ projectId: "project-1",
+ view: "OBSERVATIONS",
+ AND: [
+ { metric: { path: ["measure"], equals: "totalCost" } },
+ { metric: { path: ["aggregation"], equals: "sum" } },
+ {
+ filters: {
+ array_contains: [
+ {
+ column: "evaluatorId",
+ type: "string",
+ operator: "is not empty",
+ value: "",
+ },
+ ],
+ },
+ },
+ {
+ filters: {
+ array_contains: [
+ {
+ column: "isEvaluatorTest",
+ type: "boolean",
+ operator: "=",
+ value: false,
+ },
+ ],
+ },
+ },
+ ],
+ },
+ }),
+ );
+ });
+});
diff --git a/web/src/features/monitors/server/listLinkedEvaluatorAlerts.ts b/web/src/features/monitors/server/listLinkedEvaluatorAlerts.ts
new file mode 100644
index 0000000000..3ff4e9169d
--- /dev/null
+++ b/web/src/features/monitors/server/listLinkedEvaluatorAlerts.ts
@@ -0,0 +1,106 @@
+import { metric as MetricSchema } from "@langfuse/shared";
+import { findMonitorIdsLinkedToEvaluators } from "@langfuse/shared/monitors/server";
+import type { Prisma, PrismaClient } from "@langfuse/shared/src/db";
+
+const LINKED_ALERTS_PAGE_SIZE = 20;
+
+/** Lists the first page of monitors connected to an evaluator alert scope. */
+export async function listLinkedEvaluatorAlerts(
+ prisma: PrismaClient,
+ params:
+ | { scope: "evaluator"; projectId: string; evaluatorId: string }
+ | { scope: "allEvaluators"; projectId: string },
+) {
+ const evaluatorMonitorIds =
+ params.scope === "evaluator"
+ ? await findMonitorIdsLinkedToEvaluators(prisma, {
+ projectId: params.projectId,
+ evaluatorIds: [params.evaluatorId],
+ limit: LINKED_ALERTS_PAGE_SIZE + 1,
+ })
+ : undefined;
+ const where: Prisma.MonitorWhereInput =
+ params.scope === "evaluator"
+ ? {
+ projectId: params.projectId,
+ id: { in: evaluatorMonitorIds },
+ }
+ : {
+ projectId: params.projectId,
+ view: "OBSERVATIONS",
+ AND: [
+ { metric: { path: ["measure"], equals: "totalCost" } },
+ { metric: { path: ["aggregation"], equals: "sum" } },
+ {
+ filters: {
+ array_contains: [
+ {
+ column: "evaluatorId",
+ type: "string",
+ operator: "is not empty",
+ value: "",
+ },
+ ],
+ },
+ },
+ {
+ filters: {
+ array_contains: [
+ {
+ column: "isEvaluatorTest",
+ type: "boolean",
+ operator: "=",
+ value: false,
+ },
+ ],
+ },
+ },
+ ],
+ };
+
+ const monitors =
+ evaluatorMonitorIds?.length === 0
+ ? []
+ : await prisma.monitor.findMany({
+ where,
+ select: {
+ id: true,
+ name: true,
+ status: true,
+ severity: true,
+ metric: true,
+ thresholdOperator: true,
+ alertThreshold: true,
+ alertedAt: true,
+ },
+ orderBy: { updatedAt: "desc" },
+ take: LINKED_ALERTS_PAGE_SIZE + 1,
+ });
+
+ return {
+ data: monitors
+ .slice(0, LINKED_ALERTS_PAGE_SIZE)
+ .map(
+ ({
+ id,
+ name,
+ status,
+ severity,
+ metric,
+ thresholdOperator,
+ alertThreshold,
+ alertedAt,
+ }) => ({
+ id,
+ name,
+ status,
+ severity,
+ metric: MetricSchema.parse(metric),
+ thresholdOperator,
+ alertThreshold: alertThreshold.toNumber(),
+ alertedAt,
+ }),
+ ),
+ hasMore: monitors.length > LINKED_ALERTS_PAGE_SIZE,
+ };
+}
diff --git a/web/src/features/monitors/server/suggestMonitorName.ts b/web/src/features/monitors/server/suggestMonitorName.ts
new file mode 100644
index 0000000000..488e62a7c0
--- /dev/null
+++ b/web/src/features/monitors/server/suggestMonitorName.ts
@@ -0,0 +1,57 @@
+import type { PrismaClient } from "@langfuse/shared/src/db";
+import {
+ ChatMessageRole,
+ ChatMessageType,
+ generateLangfuseAIText,
+ getClientInitiatedNonStreamingLlmTimeoutMs,
+ logger,
+} from "@langfuse/shared/src/server";
+
+import { resolveLangfuseAiFeatureAvailability } from "@/src/features/ai-features/server";
+
+/** Generates a concise monitor name when Langfuse AI features are available. */
+export async function suggestMonitorName({
+ prisma,
+ projectId,
+ description,
+}: {
+ prisma: PrismaClient;
+ projectId: string;
+ description: string;
+}): Promise<string | null> {
+ const availability = await resolveLangfuseAiFeatureAvailability({
+ prisma,
+ projectId,
+ });
+ if (!availability.available) return null;
+
+ try {
+ const generated = await generateLangfuseAIText({
+ messages: [
+ {
+ role: ChatMessageRole.System,
+ content:
+ "Return only a concise alert title of at most six words. Describe what is monitored and the trigger. Do not use quotes or punctuation at the end.",
+ type: ChatMessageType.System,
+ },
+ {
+ role: ChatMessageRole.User,
+ content: description,
+ type: ChatMessageType.User,
+ },
+ ],
+ model: availability.model,
+ maxTokens: 40,
+ timeout: getClientInitiatedNonStreamingLlmTimeoutMs(),
+ });
+ return (
+ generated
+ ?.trim()
+ .replace(/^['"]|['"]$/g, "")
+ .slice(0, 200) || null
+ );
+ } catch (error) {
+ logger.warn("Alert title generation failed", { projectId, error });
+ return null;
+ }
+}
diff --git a/web/src/features/organizations/index.ts b/web/src/features/organizations/index.ts
new file mode 100644
index 0000000000..23e38ca77e
--- /dev/null
+++ b/web/src/features/organizations/index.ts
@@ -0,0 +1 @@
+export { useLangfuseCloudRegion } from "@/src/features/organizations/hooks";
diff --git a/web/src/features/posthog-analytics/usePostHogClientCapture.ts b/web/src/features/posthog-analytics/usePostHogClientCapture.ts
index ca1e7f6b81..16e9a424f0 100644
--- a/web/src/features/posthog-analytics/usePostHogClientCapture.ts
+++ b/web/src/features/posthog-analytics/usePostHogClientCapture.ts
@@ -185,6 +185,7 @@ const events = {
"empty_state_template_select",
"empty_state_browse_library",
"empty_state_detect_topics",
+ "alert_create_clicked",
],
evaluation_rules: [
"create",
@@ -245,7 +246,7 @@ const events = {
"delete_dashboard_form_open",
"delete_dashboard_button_click",
],
- monitors: ["delete_form_open", "delete_monitor_button_click"],
+ monitors: ["create", "delete_form_open", "delete_monitor_button_click"],
datasets: [
"delete_form_open",
"delete_dataset_button_click",
diff --git a/web/src/features/projects/index.ts b/web/src/features/projects/index.ts
new file mode 100644
index 0000000000..1e0fac144a
--- /dev/null
+++ b/web/src/features/projects/index.ts
@@ -0,0 +1 @@
+export { useProject } from "@/src/features/projects/hooks";
diff --git a/web/src/features/widgets/components/WidgetForm.observationNameOptions.clienttest.tsx b/web/src/features/widgets/components/WidgetForm.observationNameOptions.clienttest.tsx
index 5ce5f21bbe..9d74829fa1 100644
--- a/web/src/features/widgets/components/WidgetForm.observationNameOptions.clienttest.tsx
+++ b/web/src/features/widgets/components/WidgetForm.observationNameOptions.clienttest.tsx
@@ -26,6 +26,7 @@ vi.mock("@/src/utils/api", () => ({
filterOptions: { useQuery: generationsFilterOptionsUseQuery },
},
events: { filterOptions: { useQuery: noopQuery } },
+ evalsV2: { options: { useQuery: noopQuery } },
projects: { environmentFilterOptions: { useQuery: noopQuery } },
datasets: { allDatasetMeta: { useQuery: noopQuery } },
dashboard: { executeQuery: { useQuery: noopQuery } },
diff --git a/web/src/server/api/routers/monitors.ts b/web/src/server/api/routers/monitors.ts
index 4b9002daed..01ae4cb7a1 100644
--- a/web/src/server/api/routers/monitors.ts
+++ b/web/src/server/api/routers/monitors.ts
@@ -7,6 +7,10 @@ import {
} from "@/src/server/api/trpc";
import { throwIfNoProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
import { createWithinEntitlementLimit } from "@/src/features/entitlements/server/createWithinEntitlementLimit";
+import {
+ listLinkedEvaluatorAlerts,
+ suggestMonitorName,
+} from "@/src/features/monitors/server";
import {
CreateMonitorSchema,
DeleteMonitorSchema,
@@ -109,6 +113,55 @@ export const monitorsRouter = createTRPCRouter({
return { count };
}),
+ suggestName: monitorsProcedure
+ .input(
+ z.object({
+ projectId: z.string(),
+ description: z.string().min(1).max(500),
+ }),
+ )
+ .mutation(async ({ ctx, input }) => {
+ throwIfNoProjectAccess({
+ session: ctx.session,
+ projectId: input.projectId,
+ scope: "alerts:CUD",
+ });
+ return suggestMonitorName({
+ prisma: ctx.prisma,
+ projectId: input.projectId,
+ description: input.description,
+ });
+ }),
+
+ linkedEvaluatorAlerts: monitorsProcedure
+ .input(z.object({ projectId: z.string(), evaluatorId: z.string() }))
+ .query(async ({ ctx, input }) => {
+ throwIfNoProjectAccess({
+ session: ctx.session,
+ projectId: input.projectId,
+ scope: "alerts:read",
+ });
+ return listLinkedEvaluatorAlerts(ctx.prisma, {
+ scope: "evaluator",
+ projectId: input.projectId,
+ evaluatorId: input.evaluatorId,
+ });
+ }),
+
+ linkedAllEvaluatorSpendAlerts: monitorsProcedure
+ .input(z.object({ projectId: z.string() }))
+ .query(async ({ ctx, input }) => {
+ throwIfNoProjectAccess({
+ session: ctx.session,
+ projectId: input.projectId,
+ scope: "alerts:read",
+ });
+ return listLinkedEvaluatorAlerts(ctx.prisma, {
+ scope: "allEvaluators",
+ projectId: input.projectId,
+ });
+ }),
+
/** hasAny reports whether the project owns at least one monitor; drives the list-page onboarding splash. */
hasAny: monitorsProcedure
.input(z.object({ projectId: z.string() }))