🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(evals): add evaluator alert management (#16903)
Tobias Wochingeradd518fe2880
diff --git a/packages/shared/src/features/monitors/server.ts b/packages/shared/src/features/monitors/server.tsindex 0027328e71..69f1e1f399 100644--- a/packages/shared/src/features/monitors/server.ts+++ b/packages/shared/src/features/monitors/server.ts@@ -3,6 +3,7 @@* scheduler DTOs. */export * from "./index";export * from "./service";+export * from "./service/evaluatorFilterMatching";export * from "./scheduler/types";export * from "./scheduler/scheduler";export * from "./processor/processor";diff --git a/packages/shared/src/features/monitors/service/evaluatorFilterMatching.ts b/packages/shared/src/features/monitors/service/evaluatorFilterMatching.tsnew file mode 100644index 0000000000..945446cc0b--- /dev/null+++ b/packages/shared/src/features/monitors/service/evaluatorFilterMatching.ts@@ -0,0 +1,64 @@+import { Prisma, type PrismaClient } from "../../../db";++/**+ * Matches an evaluator ID against the positive evaluator filter operators+ * that can keep a monitor linked to that evaluator.+ */+export const evaluatorFilterMatchesSql = (+ filterRow: Prisma.Sql,+ evaluatorId: Prisma.Sql,+): Prisma.Sql => Prisma.sql`+ ${filterRow}->>'column' = 'evaluatorId'+ AND (+ (+ ${filterRow}->>'type' = 'string'+ AND (+ (${filterRow}->>'operator' = '=' AND ${evaluatorId} = ${filterRow}->>'value')+ OR (${filterRow}->>'operator' = 'contains' AND strpos(${evaluatorId}, ${filterRow}->>'value') > 0)+ OR (${filterRow}->>'operator' = 'starts with' AND starts_with(${evaluatorId}, ${filterRow}->>'value'))+ )+ )+ OR (+ ${filterRow}->>'type' = 'stringOptions'+ AND ${filterRow}->>'operator' = 'any of'+ AND ${filterRow}->'value' @> jsonb_build_array(${evaluatorId})+ )+ )+`;++/** Returns monitor IDs whose evaluator filter matches any supplied evaluator. */+export const findMonitorIdsLinkedToEvaluators = async (+ client: Pick<PrismaClient, "$queryRaw">,+ params: {+ projectId: string;+ evaluatorIds: string[];+ limit?: number;+ },+): Promise<string[]> => {+ if (params.evaluatorIds.length === 0) return [];++ const selectedEvaluators = Prisma.join(+ params.evaluatorIds.map((id) => Prisma.sql`(${id})`),+ );+ const matchesEvaluator = evaluatorFilterMatchesSql(+ Prisma.raw("filter_row"),+ Prisma.raw("selected_evaluator.id"),+ );+ const limit = params.limit ? Prisma.sql`LIMIT ${params.limit}` : Prisma.empty;++ const rows = await client.$queryRaw<{ id: string }[]>(Prisma.sql`+ SELECT monitor.id+ FROM monitors AS monitor+ WHERE monitor.project_id = ${params.projectId}+ AND EXISTS (+ SELECT 1+ FROM jsonb_array_elements(monitor.filters) AS filter_row+ CROSS JOIN (VALUES ${selectedEvaluators}) AS selected_evaluator(id)+ WHERE ${matchesEvaluator}+ )+ ORDER BY monitor.updated_at DESC+ ${limit}+ `);++ return rows.map(({ id }) => id);+};diff --git a/packages/shared/src/features/monitors/service/helpers.test.ts b/packages/shared/src/features/monitors/service/helpers.test.tsindex ab08549908..ff1bf57591 100644--- a/packages/shared/src/features/monitors/service/helpers.test.ts+++ b/packages/shared/src/features/monitors/service/helpers.test.ts@@ -184,6 +184,48 @@ describe("toPrismaWhere", () => {});});+ it("translates evaluator IDs into monitor-filter JSON predicates", () => {+ const filter: ListMonitorFilter = [+ {+ type: "stringOptions",+ column: "evaluatorId",+ operator: "any of",+ value: ["eval-1", "eval-2"],+ },+ ];++ expect(+ toPrismaWhere("proj_01", filter, ["monitor-1", "monitor-2"]),+ ).toEqual({+ projectId: "proj_01",+ AND: [+ {+ id: { in: ["monitor-1", "monitor-2"] },+ },+ ],+ });+ });++ it("negates excluded evaluator IDs", () => {+ const filter: ListMonitorFilter = [+ {+ type: "stringOptions",+ column: "evaluatorId",+ operator: "none of",+ value: ["eval-1"],+ },+ ];++ expect(toPrismaWhere("proj_01", filter, ["monitor-1"])).toEqual({+ projectId: "proj_01",+ AND: [+ {+ id: { notIn: ["monitor-1"] },+ },+ ],+ });+ });+it("translates tags operators to Prisma array predicates", () => {const filter: ListMonitorFilter = [{diff --git a/packages/shared/src/features/monitors/service/helpers.ts b/packages/shared/src/features/monitors/service/helpers.tsindex 6d0b808d62..25e5ae9bef 100644--- a/packages/shared/src/features/monitors/service/helpers.ts+++ b/packages/shared/src/features/monitors/service/helpers.ts@@ -55,9 +55,18 @@ export const toPrismaOrderBy = (export const toPrismaWhere = (projectId: string,filter: ListMonitorFilter | undefined,+ evaluatorMonitorIds?: string[],): Prisma.MonitorWhereInput => {const and: Prisma.MonitorWhereInput[] = [];for (const f of filter ?? []) {+ if (f.column === "evaluatorId") {+ and.push(+ f.operator === "any of"+ ? { id: { in: evaluatorMonitorIds ?? [] } }+ : { id: { notIn: evaluatorMonitorIds ?? [] } },+ );+ continue;+ }if (f.type === "stringOptions") {and.push(f.operator === "any of"diff --git a/packages/shared/src/features/monitors/service/service.ts b/packages/shared/src/features/monitors/service/service.tsindex e64a76b7cd..555198f53f 100644--- a/packages/shared/src/features/monitors/service/service.ts+++ b/packages/shared/src/features/monitors/service/service.ts@@ -28,6 +28,10 @@ import {type SessionContext,type UpdateMonitor,} from "./types";+import {+ evaluatorFilterMatchesSql,+ findMonitorIdsLinkedToEvaluators,+} from "./evaluatorFilterMatching";/*** MonitorService manages all of the Monitors that produce MonitorAlerts on@@ -161,7 +165,20 @@ export class MonitorService {): Promise<{ monitors: Monitor[]; totalCount: number }> {const skip =input.page && input.limit ? (input.page - 1) * input.limit : undefined;- const where = toPrismaWhere(input.projectId, input.filter);+ const evaluatorFilter = input.filter?.find(+ (filter) => filter.column === "evaluatorId",+ );+ const evaluatorMonitorIds = evaluatorFilter+ ? await findMonitorIdsLinkedToEvaluators(prisma, {+ projectId: input.projectId,+ evaluatorIds: evaluatorFilter.value,+ })+ : undefined;+ const where = toPrismaWhere(+ input.projectId,+ input.filter,+ evaluatorMonitorIds,+ );const [monitors, totalCount] = await Promise.all([prisma.monitor.findMany({@@ -179,15 +196,36 @@ export class MonitorService {public static async getFilterOptions(_session: SessionContext,input: GetMonitorFilterOptions,- ): Promise<{ tags: { value: string }[] }> {- const rows = await prisma.$queryRaw<{ value: string }[]>`- SELECT tags.tag AS value- FROM monitors, UNNEST(monitors.tags) AS tags(tag)- WHERE monitors.project_id = ${input.projectId}- GROUP BY tags.tag- ORDER BY tags.tag ASC;- `;- return { tags: rows };+ ): Promise<{+ tags: { value: string }[];+ evaluators: { value: string; displayValue: string }[];+ }> {+ const evaluatorFilterMatch = evaluatorFilterMatchesSql(+ Prisma.raw("filter_row"),+ Prisma.raw("e.id"),+ );+ const [tags, evaluators] = await Promise.all([+ prisma.$queryRaw<{ value: string }[]>`+ SELECT tags.tag AS value+ FROM monitors, UNNEST(monitors.tags) AS tags(tag)+ WHERE monitors.project_id = ${input.projectId}+ GROUP BY tags.tag+ ORDER BY tags.tag ASC;+ `,+ prisma.$queryRaw<{ value: string; displayValue: string }[]>`+ SELECT DISTINCT+ e.id AS value,+ e.name AS "displayValue"+ FROM monitors AS m+ CROSS JOIN LATERAL jsonb_array_elements(m.filters) AS filter_row+ JOIN evaluators AS e+ ON e.project_id = m.project_id+ AND ${evaluatorFilterMatch}+ WHERE m.project_id = ${input.projectId}+ ORDER BY "displayValue" ASC, value ASC;+ `,+ ]);+ return { tags, evaluators };}public static async delete(diff --git a/packages/shared/src/features/monitors/service/types.ts b/packages/shared/src/features/monitors/service/types.tsindex c59e0dbfd9..f58e4d26a4 100644--- a/packages/shared/src/features/monitors/service/types.ts+++ b/packages/shared/src/features/monitors/service/types.ts@@ -106,6 +106,17 @@ export type ListMonitorSeverityFilter = z.infer<typeof ListMonitorSeverityFilterSchema>;+/** ListMonitorEvaluatorIdFilterSchema matches alerts connected to selected evaluators. */+export const ListMonitorEvaluatorIdFilterSchema = z.object({+ type: z.literal("stringOptions"),+ column: z.literal("evaluatorId"),+ operator: z.enum(["any of", "none of"]),+ value: z.array(z.string()).min(1),+});+export type ListMonitorEvaluatorIdFilter = z.infer<+ typeof ListMonitorEvaluatorIdFilterSchema+>;+/** ListMonitorTagsFilterSchema is the tags row of a ListMonitorFilter. */export const ListMonitorTagsFilterSchema = z.object({type: z.literal("arrayOptions"),@@ -124,6 +135,7 @@ export const ListMonitorFilterSchema = z.array(z.discriminatedUnion("column", [ListMonitorSeverityFilterSchema,+ ListMonitorEvaluatorIdFilterSchema,ListMonitorTagsFilterSchema,]),)diff --git a/packages/shared/src/features/query/dataModel.ts b/packages/shared/src/features/query/dataModel.tsindex 2a20ff147d..1d3828f4ac 100644--- a/packages/shared/src/features/query/dataModel.ts+++ b/packages/shared/src/features/query/dataModel.ts@@ -13,6 +13,7 @@ import {eventsTableTraceNameSqlForAlias,} from "../../eventsTable";import { LISTABLE_SCORE_TYPES } from "../../domain/scores";+import { EvalExecutionMetadataKey } from "../evals/evalExecutionMetadata";// The data model defines all available dimensions, measures, and the timeDimension for a given view.// Make sure to update web/src/features/dashboard/lib/dashboardUiTableToViewMapping.ts if you make changes@@ -369,6 +370,27 @@ export const eventsTracesView: ViewDeclarationType = {baseCte: `events_core events_traces`,};+const createEvaluatorDimensions = (+ tableAlias: string,+): DimensionsDeclarationType => ({+ evaluatorId: {+ sql: `coalesce(nullIf(${tableAlias}.evaluator_id, ''), ${tableAlias}.evaluation_rule_id)`,+ alias: "evaluatorId",+ type: "string",+ description:+ "Identifier of the evaluator, falling back to its legacy evaluation rule identifier.",+ highCardinality: true,+ uiHidden: true,+ },+ isEvaluatorTest: {+ sql: `${tableAlias}.evaluator_execution_is_test`,+ alias: "isEvaluatorTest",+ type: "boolean",+ description: "Whether this row belongs to an evaluator test run.",+ uiHidden: true,+ },+});+export const observationsView: ViewDeclarationType = {name: "observations",description:@@ -903,6 +925,13 @@ const createScoreSpecificDimensions = (highCardinality: true,uiHidden: true,},+ isEvaluatorTest: {+ sql: `toBool(${tableAlias}.metadata['${EvalExecutionMetadataKey.EVALUATOR_TEST}'] = 'true')`,+ alias: "isEvaluatorTest",+ type: "boolean",+ description: "Whether this score belongs to an evaluator test run.",+ uiHidden: true,+ },source: {sql: `${tableAlias}.source`,alias: "source",@@ -1224,6 +1253,7 @@ export const eventsObservationsView: ViewDeclarationType = {highCardinality: true,uiHidden: true,},+ ...createEvaluatorDimensions("events_observations"),traceId: {sql: "events_observations.trace_id",alias: "traceId",diff --git a/packages/shared/src/features/query/server/queryBuilder.filterValidation.test.ts b/packages/shared/src/features/query/server/queryBuilder.filterValidation.test.tsindex afadf42dfd..5ff42e84bf 100644--- a/packages/shared/src/features/query/server/queryBuilder.filterValidation.test.ts+++ b/packages/shared/src/features/query/server/queryBuilder.filterValidation.test.ts@@ -314,6 +314,57 @@ describe("queryBuilder filter type validation", () => {},);+ it.each(["v1", "v2"] as const)(+ "lowers evaluator test filters in the %s scores view",+ async (version) => {+ const { query } = await buildQueryWithFilter(+ {+ column: "isEvaluatorTest",+ operator: "=",+ value: false,+ type: "boolean",+ },+ { view: "scores-numeric" },+ version,+ );++ expect(query).toContain("scores_numeric.metadata['evaluator_test']");+ expect(query).toContain(": Boolean}");+ },+ );++ it("lowers evaluator identity and test filters in the v2 observations view", async () => {+ const evaluatorIdQuery = await buildQueryWithFilter(+ {+ column: "evaluatorId",+ operator: "is not empty",+ value: "",+ type: "string",+ },+ { view: "observations" },+ "v2",+ );+ const evaluatorTestQuery = await buildQueryWithFilter(+ {+ column: "isEvaluatorTest",+ operator: "=",+ value: false,+ type: "boolean",+ },+ { view: "observations" },+ "v2",+ );++ expect(evaluatorIdQuery.query).toContain(+ "coalesce(nullIf(events_observations.evaluator_id, ''), events_observations.evaluation_rule_id)",+ );+ expect(evaluatorTestQuery.query).toContain(+ "events_observations.evaluator_execution_is_test",+ );+ expect(evaluatorIdQuery.query).toContain("!= ''");+ expect(evaluatorTestQuery.query).toContain(": Boolean}");+ });+it("lowers the semantic-root observation filter only in the v2 events view", async () => {const { query } = await buildQueryWithFilter({diff --git a/packages/shared/src/interfaces/filters.ts b/packages/shared/src/interfaces/filters.tsindex d84ab3743c..ac59691a60 100644--- a/packages/shared/src/interfaces/filters.ts+++ b/packages/shared/src/interfaces/filters.ts@@ -3,7 +3,14 @@ import { z } from "zod";// Make sure to update the InMemoryFilterService if you add new filter typesexport const filterOperators = {datetime: [">", "<", ">=", "<="],- string: ["=", "contains", "does not contain", "starts with", "ends with"],+ string: [+ "=",+ "contains",+ "does not contain",+ "starts with",+ "ends with",+ "is not empty",+ ],stringOptions: ["any of", "none of"],categoryOptions: ["any of", "none of"],arrayOptions: ["any of", "none of", "all of"],@@ -71,7 +78,7 @@ export const stringObjectFilter = z.object({type: z.literal("stringObject"),column: z.string(),key: z.string(), // eg metadata --> "environment"- operator: z.enum(filterOperators.string),+ operator: z.enum(filterOperators.stringObject),value: z.string(),});export const numberObjectFilter = z.object({diff --git a/packages/shared/src/server/filterToPrisma.ts b/packages/shared/src/server/filterToPrisma.tsindex 574a154653..4a315bdc08 100644--- a/packages/shared/src/server/filterToPrisma.ts+++ b/packages/shared/src/server/filterToPrisma.ts@@ -124,6 +124,10 @@ export function tableColumnsToSqlFilter(logger.warn("Position-in-trace filters are not supported in postgres");throw new Error("Position-in-trace filters not supported in postgres");}+ if (filter.type === "string" && filter.operator === "is not empty") {+ return Prisma.sql`(${filterAndColumn.internalColumn} IS NOT NULL AND ${filterAndColumn.internalColumn} <> '')`;+ }+const jsonKeyPrisma =filter.type === "stringObject" || filter.type === "numberObject"? Prisma.sql`->>${filter.key}`diff --git a/packages/shared/src/server/prismaFilter.ts b/packages/shared/src/server/prismaFilter.tsindex 7441406d72..89dbdb30d6 100644--- a/packages/shared/src/server/prismaFilter.ts+++ b/packages/shared/src/server/prismaFilter.ts@@ -46,6 +46,8 @@ export function stringFilterToPrisma(return { startsWith: filter.value, ...insensitive };case "ends with":return { endsWith: filter.value, ...insensitive };+ case "is not empty":+ return { not: "" };}}diff --git a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.tsindex 0842311b47..71d8aded1a 100644--- a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts+++ b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts@@ -107,6 +107,9 @@ export class StringFilter implements Filter {case "ends with":query = `endsWith(${fieldWithPrefix}, {${varName}: String})`;break;+ case "is not empty":+ query = `(${fieldWithPrefix} != '' AND ${fieldWithPrefix} IS NOT NULL)`;+ break;case FTS_MATCH_OPERATOR:assertValidFtsMatchFilter({filterType: "string",@@ -132,8 +135,8 @@ export class StringFilter implements Filter {}return {- query: query,- params: { [varName]: this.value },+ query,+ params: this.operator === "is not empty" ? {} : { [varName]: this.value },};}}diff --git a/packages/shared/src/server/services/InMemoryFilterService.ts b/packages/shared/src/server/services/InMemoryFilterService.tsindex cc9fdc892b..53463e7b43 100644--- a/packages/shared/src/server/services/InMemoryFilterService.ts+++ b/packages/shared/src/server/services/InMemoryFilterService.ts@@ -143,6 +143,8 @@ export class InMemoryFilterService {return strValue.startsWith(filterValue);case "ends with":return strValue.endsWith(filterValue);+ case "is not empty":+ return strValue.length > 0;default:logger.error("Unsupported string filter operator", {operator,diff --git a/packages/shared/src/tableDefinitions/monitorsTable.ts b/packages/shared/src/tableDefinitions/monitorsTable.tsindex 936c0a0c68..5e4082363b 100644--- a/packages/shared/src/tableDefinitions/monitorsTable.ts+++ b/packages/shared/src/tableDefinitions/monitorsTable.ts@@ -34,4 +34,11 @@ export const monitorsTableCols: ColumnDefinition[] = [internal: 'm."tags"',options: [],},+ {+ name: "Evaluator ID",+ id: "evaluatorId",+ type: "stringOptions",+ internal: "evaluatorId",+ options: [],+ },];diff --git a/packages/shared/src/tableDefinitions/types.ts b/packages/shared/src/tableDefinitions/types.tsindex 0a3a61e95e..8203c4e496 100644--- a/packages/shared/src/tableDefinitions/types.ts+++ b/packages/shared/src/tableDefinitions/types.ts@@ -64,6 +64,8 @@ export type ColumnDefinition =step?: number;/** Minimum value for number inputs. */min?: number;+ /** Optional suggestions whose values are persisted while displayValue is rendered. */+ options?: Array<SingleValueOption>;}| {name: string;diff --git a/web/src/__tests__/server/monitors.servertest.ts b/web/src/__tests__/server/monitors.servertest.tsindex 84293b9121..818fc03958 100644--- a/web/src/__tests__/server/monitors.servertest.ts+++ b/web/src/__tests__/server/monitors.servertest.ts@@ -167,6 +167,118 @@ describe("monitors trpc", () => {expect(list.monitors.map((m) => m.id)).toContain(created.id);});+ it("does not suggest a title when AI features are unavailable", async () => {+ const { project, caller } = await prepare();++ await expect(+ caller.monitors.suggestName({+ projectId: project.id,+ description: "Count of observations is above 100",+ }),+ ).resolves.toBeNull();+ });++ it.each([+ {+ name: "equals",+ filter: {+ column: "evaluatorId",+ type: "string" as const,+ operator: "=" as const,+ value: "evaluator-1",+ },+ },+ {+ name: "contains",+ filter: {+ column: "evaluatorId",+ type: "string" as const,+ operator: "contains" as const,+ value: "valuator-1",+ },+ },+ {+ name: "starts with",+ filter: {+ column: "evaluatorId",+ type: "string" as const,+ operator: "starts with" as const,+ value: "evaluator-",+ },+ },+ {+ name: "any of",+ filter: {+ column: "evaluatorId",+ type: "stringOptions" as const,+ operator: "any of" as const,+ value: ["evaluator-2", "evaluator-1"],+ },+ },+ ])(+ "returns alerts whose $name filter matches the evaluator id",+ async ({ name, filter }) => {+ const { project, caller } = await prepare();+ const monitor = await caller.monitors.create({+ ...validMonitorInput(project.id),+ name: `${name} evaluator`,+ filters: [filter],+ });++ const linked = await caller.monitors.linkedEvaluatorAlerts({+ projectId: project.id,+ evaluatorId: "evaluator-1",+ });++ expect(linked.data.map(({ id }) => id)).toEqual([monitor.id]);+ expect(linked.hasMore).toBe(false);+ },+ );++ it("returns aggregate evaluator spend alerts", async () => {+ const { project, caller } = await prepare();++ const aggregate = await caller.monitors.create({+ ...validMonitorInput(project.id),+ name: "All evaluator spend",+ metric: { measure: "totalCost", aggregation: "sum" },+ filters: [+ {+ column: "evaluatorId",+ type: "string",+ operator: "is not empty",+ value: "",+ },+ {+ column: "isEvaluatorTest",+ type: "boolean",+ operator: "=",+ value: false,+ },+ ],+ });++ await expect(+ caller.monitors.linkedAllEvaluatorSpendAlerts({+ projectId: project.id,+ }),+ ).resolves.toEqual({+ data: [+ {+ id: aggregate.id,+ name: "All evaluator spend",+ status: MonitorStatusSchema.enum.ACTIVE,+ severity: MonitorSeveritySchema.enum.UNKNOWN,+ metric: { measure: "totalCost", aggregation: "sum" },+ thresholdOperator: MonitorThresholdOperatorSchema.enum.GT,+ alertThreshold: 100,+ alertedAt: null,+ },+ ],+ hasMore: false,+ });+ });+it("returns ERROR_BAD_QUERY monitors verbatim (scheduler-owned status survives a read)", async () => {const { project, caller } = await prepare();@@ -207,7 +319,7 @@ describe("monitors trpc", () => {expect(created.id).toBeDefined();});- it("allows monitors.all from VIEWER role (read-only scope)", async () => {+ it("allows monitor reads from VIEWER role (read-only scope)", async () => {const { project, caller } = await prepare({ projectRole: "VIEWER" });const list = await caller.monitors.all({@@ -216,7 +328,16 @@ describe("monitors trpc", () => {page: 1,limit: 50,});+ const linkedAlerts = await caller.monitors.linkedEvaluatorAlerts({+ projectId: project.id,+ evaluatorId: "evaluator-1",+ });+ const spendAlerts = await caller.monitors.linkedAllEvaluatorSpendAlerts({+ projectId: project.id,+ });expect(list.totalCount).toBe(0);+ expect(linkedAlerts).toEqual({ data: [], hasMore: false });+ expect(spendAlerts).toEqual({ data: [], hasMore: false });});});@@ -415,12 +536,38 @@ describe("monitors trpc", () => {expect(result.monitors.map((m) => m.name)).toEqual(["Tagged"]);});- it("filterOptions returns distinct tags for the project", async () => {+ it("filterOptions returns tags and evaluators with specific alerts", async () => {const { project, caller } = await prepare();+ const linkedEvaluatorId = v4();+ const unlinkedEvaluatorId = v4();+ await prisma.evaluator.createMany({+ data: [+ {+ id: linkedEvaluatorId,+ projectId: project.id,+ name: "Linked evaluator",+ type: "CODE",+ },+ {+ id: unlinkedEvaluatorId,+ projectId: project.id,+ name: "Unlinked evaluator",+ type: "CODE",+ },+ ],+ });await caller.monitors.create({...validMonitorInput(project.id),name: "A",tags: ["prod", "latency"],+ filters: [+ {+ column: "evaluatorId",+ type: "string",+ operator: "contains",+ value: linkedEvaluatorId.slice(1, -1),+ },+ ],});await caller.monitors.create({...validMonitorInput(project.id),@@ -432,6 +579,25 @@ describe("monitors trpc", () => {projectId: project.id,});expect(opts.tags.map((t) => t.value).sort()).toEqual(["latency", "prod"]);+ expect(opts.evaluators).toEqual([+ { value: linkedEvaluatorId, displayValue: "Linked evaluator" },+ ]);++ const filtered = await caller.monitors.all({+ projectId: project.id,+ orderBy: null,+ page: 1,+ limit: 50,+ filter: [+ {+ type: "stringOptions",+ column: "evaluatorId",+ operator: "any of",+ value: [linkedEvaluatorId],+ },+ ],+ });+ expect(filtered.monitors.map((monitor) => monitor.name)).toEqual(["A"]);});});diff --git a/web/src/features/ai-features/server/index.ts b/web/src/features/ai-features/server/index.tsnew file mode 100644index 0000000000..b6fc50ad56--- /dev/null+++ b/web/src/features/ai-features/server/index.ts@@ -0,0 +1 @@+export { resolveLangfuseAiFeatureAvailability } from "@/src/features/ai-features/server/availability";diff --git a/web/src/features/dashboard/lib/dashboardUiTableToViewMapping.ts b/web/src/features/dashboard/lib/dashboardUiTableToViewMapping.tsindex b239bee115..2c9248e9e5 100644--- a/web/src/features/dashboard/lib/dashboardUiTableToViewMapping.ts+++ b/web/src/features/dashboard/lib/dashboardUiTableToViewMapping.ts@@ -56,6 +56,23 @@ const defineField = (legacy,});+const evaluatorFilterDefinitions = [+ defineField(+ "evaluatorId",+ sourceSpec("Evaluator", {+ uiTableId: "evaluatorId",+ aliases: ["Evaluator ID"],+ }),+ ),+ defineField(+ "isEvaluatorTest",+ sourceSpec("Evaluator test run", {+ uiTableId: "isEvaluatorTest",+ aliases: ["Evaluator execution"],+ }),+ ),+] as const;+const viewFilterDefinitions: Record<ViewName,readonly DashboardViewFieldDefinition[]@@ -145,6 +162,7 @@ const viewFilterDefinitions: Record<"environment",sourceSpec("Environment", { uiTableId: "environment" }),),+ ...evaluatorFilterDefinitions,defineField("release",sourceSpec("Release", {@@ -220,6 +238,7 @@ const viewFilterDefinitions: Record<"traceVersion",sourceSpec("Version", { uiTableId: "version" }),),+ ...evaluatorFilterDefinitions,],"scores-boolean": [defineField("name", sourceSpec("Score Name", { uiTableId: "scoreName" })),@@ -267,6 +286,7 @@ const viewFilterDefinitions: Record<"traceVersion",sourceSpec("Version", { uiTableId: "version" }),),+ ...evaluatorFilterDefinitions,],"scores-categorical": [defineField("name", sourceSpec("Score Name", { uiTableId: "scoreName" })),@@ -314,6 +334,7 @@ const viewFilterDefinitions: Record<"traceVersion",sourceSpec("Version", { uiTableId: "version" }),),+ ...evaluatorFilterDefinitions,],};diff --git a/web/src/features/entitlements/index.ts b/web/src/features/entitlements/index.tsnew file mode 100644index 0000000000..172f4efcc2--- /dev/null+++ b/web/src/features/entitlements/index.ts@@ -0,0 +1 @@+export { useEntitlementLimit } from "@/src/features/entitlements/hooks";diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorAlertButton/EvaluatorAlertButton.clienttest.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorAlertButton/EvaluatorAlertButton.clienttest.tsxnew file mode 100644index 0000000000..ff27460a19--- /dev/null+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorAlertButton/EvaluatorAlertButton.clienttest.tsx@@ -0,0 +1,191 @@+import { fireEvent, render, screen } from "@testing-library/react";++import { EvaluatorAlertButton } from "./EvaluatorAlertButton";++const routerPush = vi.hoisted(() => vi.fn());++vi.mock("next/router", () => ({+ useRouter: () => ({ push: routerPush }),+}));++vi.mock("@/src/features/posthog-analytics", () => ({+ usePostHogClientCapture: () => vi.fn(),+}));++describe("EvaluatorAlertButton", () => {+ beforeAll(() => {+ vi.stubGlobal(+ "ResizeObserver",+ class {+ observe() {}+ unobserve() {}+ disconnect() {}+ },+ );+ });++ afterAll(() => {+ vi.unstubAllGlobals();+ });++ beforeEach(() => {+ routerPush.mockReset().mockResolvedValue(true);+ });++ it("offers to add an alert when none are configured", () => {+ const { container } = render(+ <EvaluatorAlertButton+ scope="allEvaluators"+ projectId="project-1"+ connectedAlerts={[]}+ canRead+ canCreate+ />,+ );++ expect(+ screen.getByRole("button", { name: "Add evaluator alert" }),+ ).toHaveTextContent("Add alert");+ expect(container.querySelector(".lucide-plus")).toBeInTheDocument();+ expect(container.querySelector(".lucide-bell")).not.toBeInTheDocument();+ });++ it("keeps the alert label and shows a spinner while loading", () => {+ const { container } = render(+ <EvaluatorAlertButton+ scope="allEvaluators"+ projectId="project-1"+ connectedAlerts={[]}+ isLoading+ canRead+ canCreate+ />,+ );++ expect(+ screen.getByRole("button", { name: "Loading evaluator alerts" }),+ ).toHaveTextContent("Add alert");+ expect(container.querySelector(".lucide-loader-circle")).toHaveClass(+ "animate-spin",+ );+ expect(container.querySelector(".lucide-plus")).not.toBeInTheDocument();+ });++ it("uses the evaluator score style for code evaluators", () => {+ render(+ <EvaluatorAlertButton+ scope="evaluator"+ projectId="project-1"+ evaluatorId="evaluator-1"+ evaluatorType="CODE"+ scoreDataType="BOOLEAN"+ connectedAlerts={[]}+ canRead+ canCreate+ />,+ );++ fireEvent.click(+ screen.getByRole("button", { name: "Add evaluator alert" }),+ );++ const scoreAlert = screen+ .getByText("Score threshold")+ .closest("[cmdk-item]");+ expect(scoreAlert).not.toBeNull();+ expect(+ scoreAlert?.querySelector(".lucide-arrow-up-right"),+ ).toBeInTheDocument();+ expect(screen.queryByText("Numeric score")).not.toBeInTheDocument();+ expect(screen.queryByText("Boolean score")).not.toBeInTheDocument();+ expect(screen.queryByText("Categorical score")).not.toBeInTheDocument();+ expect(screen.queryByText("Cost threshold")).not.toBeInTheDocument();+ });++ it("does not offer another cost alert for connected code evaluators", () => {+ render(+ <EvaluatorAlertButton+ scope="evaluator"+ projectId="project-1"+ evaluatorId="evaluator-1"+ evaluatorType="CODE"+ scoreDataType="BOOLEAN"+ connectedAlerts={[+ {+ id: "alert-1",+ name: "Evaluator alert",+ status: "ACTIVE",+ severity: "UNKNOWN",+ metric: { measure: "count", aggregation: "count" },+ thresholdOperator: "GT",+ alertThreshold: 1,+ alertedAt: null,+ },+ ]}+ canRead+ canCreate+ />,+ );++ fireEvent.click(+ screen.getByRole("button", { name: "1 connected evaluator alert" }),+ );++ expect(screen.getByRole("button", { name: "Score" })).toBeInTheDocument();+ expect(+ screen.queryByRole("button", { name: "Cost" }),+ ).not.toBeInTheDocument();+ });++ it("shows a table link after 20 connected alerts and keeps bottom actions icon-free", () => {+ render(+ <EvaluatorAlertButton+ scope="evaluator"+ projectId="project-1"+ evaluatorId="evaluator-1"+ evaluatorType="LLM_AS_JUDGE"+ scoreDataType="NUMERIC"+ connectedAlerts={[+ {+ id: "alert-1",+ name: "Evaluator alert",+ status: "ACTIVE",+ severity: "UNKNOWN",+ metric: { measure: "count", aggregation: "count" },+ thresholdOperator: "GT",+ alertThreshold: 1,+ alertedAt: null,+ },+ ]}+ hasMore+ canRead+ canCreate+ />,+ );++ fireEvent.click(+ screen.getByRole("button", { name: "1 connected evaluator alert" }),+ );++ const connectedAlert = screen+ .getByText("Evaluator alert")+ .closest("[cmdk-item]");+ expect(+ connectedAlert?.querySelector(".lucide-arrow-up-right"),+ ).toBeInTheDocument();++ const scoreAction = screen.getByRole("button", { name: "Score" });+ const costAction = screen.getByRole("button", { name: "Cost" });+ expect(+ scoreAction.querySelector(".lucide-arrow-up-right"),+ ).not.toBeInTheDocument();+ expect(+ costAction.querySelector(".lucide-arrow-up-right"),+ ).not.toBeInTheDocument();++ fireEvent.click(screen.getByText("See all alerts"));+ expect(routerPush).toHaveBeenCalledWith(+ "/project/project-1/alerts?filter=evaluatorId%3BstringOptions%3B%3Bany+of%3Bevaluator-1",+ );+ });+});diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorAlertButton/EvaluatorAlertButton.stories.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorAlertButton/EvaluatorAlertButton.stories.tsxnew file mode 100644index 0000000000..dc5f076c9e--- /dev/null+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorAlertButton/EvaluatorAlertButton.stories.tsx@@ -0,0 +1,116 @@+import preview from "../../../../../../../.storybook/preview";+import { EvaluatorAlertButton } from "./EvaluatorAlertButton";++const meta = preview.meta({ component: EvaluatorAlertButton });++const alertDefaults = {+ status: "ACTIVE",+ metric: { measure: "value", aggregation: "avg" },+ thresholdOperator: "LT",+ alertThreshold: 0.7,+ alertedAt: new Date(Date.now() - 2 * 24 * 60 * 60 * 1000),+} as const;++const defaultArgs = {+ projectId: "project-1",+ scope: "evaluator" as const,+ evaluatorId: "evaluator-1",+ evaluatorType: "LLM_AS_JUDGE" as const,+ scoreDataType: "NUMERIC" as const,+ connectedAlerts: [],+ canRead: true,+ canCreate: true,+};++export const NoAlerts = meta.story({ args: defaultArgs });++export const ConnectedAlerts = meta.story({+ args: {+ ...defaultArgs,+ connectedAlerts: [+ {+ ...alertDefaults,+ id: "alert-1",+ name: "Accuracy dropped",+ severity: "ALERT",+ },+ {+ ...alertDefaults,+ id: "alert-2",+ name: "Evaluator spend increased",+ severity: "WARNING",+ metric: { measure: "totalCost", aggregation: "sum" },+ thresholdOperator: "GT",+ alertThreshold: 10,+ },+ {+ ...alertDefaults,+ id: "alert-3",+ name: "Low quality score",+ severity: "OK",+ },+ ],+ },+});++export const CodeEvaluator = meta.story({+ args: {+ ...defaultArgs,+ evaluatorType: "CODE",+ scoreDataType: "BOOLEAN",+ },+});++export const LoadingSpinner = meta.story({+ args: { ...defaultArgs, isLoading: true },+});++export const ReadOnly = meta.story({+ args: {+ ...defaultArgs,+ connectedAlerts: [+ {+ ...alertDefaults,+ id: "alert-1",+ name: "Accuracy dropped",+ status: "PAUSED",+ severity: "PAUSED",+ },+ ],+ canCreate: false,+ },+});++export const LimitReached = meta.story({+ args: { ...defaultArgs, limitReached: true },+});++export const AggregateCost = meta.story({+ args: {+ projectId: "project-1",+ scope: "allEvaluators",+ connectedAlerts: [],+ canRead: true,+ canCreate: true,+ },+});++export const AggregateCostConnected = meta.story({+ args: {+ projectId: "project-1",+ scope: "allEvaluators",+ connectedAlerts: [+ {+ ...alertDefaults,+ id: "alert-1",+ name: "Weekly evaluator cost",+ severity: "OK",+ metric: { measure: "totalCost", aggregation: "sum" },+ thresholdOperator: "GT",+ alertThreshold: 50,+ },+ ],+ canRead: true,+ canCreate: true,+ },+});diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorAlertButton/EvaluatorAlertButton.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorAlertButton/EvaluatorAlertButton.tsxnew file mode 100644index 0000000000..0c153a20d9--- /dev/null+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorAlertButton/EvaluatorAlertButton.tsx@@ -0,0 +1,339 @@+import { useRouter } from "next/router";+import {+ ArrowUpRight,+ Bell,+ ChevronDown,+ DollarSign,+ Gauge,+ LoaderCircle,+ Plus,+} from "lucide-react";++import { Badge } from "@/src/components/ui/badge";+import { Button } from "@/src/components/ui/button";+import {+ Command,+ CommandGroup,+ CommandItem,+ CommandList,+ CommandSeparator,+} from "@/src/components/ui/command";+import { PopoverController } from "@/src/components/ui/popover";+import {+ evaluatorAlertsListUrl,+ evaluatorAlertUrl,+} from "@/src/features/evals/v2/fns/evaluators/evaluatorAlertUrl";+import { MonitorSeverityBadge } from "@/src/features/monitors";+import { usePostHogClientCapture } from "@/src/features/posthog-analytics";+import { renderEvaluatorAlertLastFired } from "@/src/features/evals/v2/fns/evaluators/renderEvaluatorAlertLastFired";+import { renderEvaluatorAlertTriggerCondition } from "@/src/features/evals/v2/fns/evaluators/renderEvaluatorAlertTriggerCondition";+import type { RouterOutputs } from "@/src/utils/api";++type ConnectedAlert =+ RouterOutputs["monitors"]["linkedEvaluatorAlerts"]["data"][number];++type EvaluatorAlertButtonProps = {+ projectId: string;+ connectedAlerts: ConnectedAlert[];+ hasMore?: boolean;+ isLoading?: boolean;+ canRead: boolean;+ canCreate: boolean;+ limitReached?: boolean;+} & (+ | {+ scope: "evaluator";+ evaluatorId: string;+ evaluatorType: "LLM_AS_JUDGE" | "CODE";+ scoreDataType?: "NUMERIC" | "BOOLEAN" | "CATEGORICAL";+ }+ | { scope: "allEvaluators" }+);++/** Alert picker for one evaluator or aggregate evaluator cost. */+export function EvaluatorAlertButton(props: EvaluatorAlertButtonProps) {+ const {+ projectId,+ connectedAlerts,+ hasMore = false,+ isLoading = false,+ canRead,+ canCreate,+ limitReached = false,+ } = props;+ const router = useRouter();+ const capture = usePostHogClientCapture();+ const createDisabled = !canCreate || limitReached;+ const disabledReason = limitReached+ ? "Alert limit reached"+ : "You do not have permission to create alerts";+ const alertCount = connectedAlerts.length;+ const isAggregateCost = props.scope === "allEvaluators";+ const supportsCostAlert =+ isAggregateCost ||+ (props.scope === "evaluator" && props.evaluatorType === "LLM_AS_JUDGE");+ const scoreAlertUrl = (dataType: "NUMERIC" | "BOOLEAN" | "CATEGORICAL") => {+ if (props.scope !== "evaluator") return "";+ return evaluatorAlertUrl(projectId, {+ type: "score",+ evaluatorId: props.evaluatorId,+ scoreDataType: dataType,+ });+ };+ const costAlertUrl = evaluatorAlertUrl(+ projectId,+ isAggregateCost+ ? { type: "allEvaluatorCost" }+ : { type: "cost", evaluatorId: props.evaluatorId },+ );+ const selectedScoreDataType =+ props.scope === "evaluator" ? props.scoreDataType : undefined;+ const navigate = (href: string, closePopover: () => void) => {+ closePopover();+ router.push(href).catch(() => undefined);+ };+ const openAlertCreation = (+ alertType: "score" | "cost" | "all_evaluator_cost",+ href: string,+ closePopover: () => void,+ dataType?: "NUMERIC" | "BOOLEAN" | "CATEGORICAL",+ ) => {+ capture("evaluators:alert_create_clicked", {+ source: isAggregateCost ? "evaluator_overview" : "evaluator_detail",+ alertType,+ scoreDataType: dataType,+ });+ navigate(href, closePopover);+ };++ return (+ <PopoverController+ align="end"+ contentClassName="w-96 p-0"+ disabled={false}+ modal={false}+ renderContent={({ closePopover }) => (+ <Command defaultValue="__none__">+ <CommandList>+ {canRead && connectedAlerts.length > 0 ? (+ <div className="p-2">+ <p className="text-muted-foreground px-2 py-1.5 text-[10px] font-bold tracking-wider uppercase">+ Connected · {connectedAlerts.length}+ {hasMore ? "+" : ""}+ </p>+ <div className="space-y-1">+ {connectedAlerts.map((alert) => (+ <CommandItem+ key={alert.id}+ value={`connected ${alert.name} ${alert.id}`}+ className="items-start rounded-md px-3 py-2.5"+ onSelect={() =>+ navigate(+ `/project/${projectId}/alerts/${alert.id}`,+ closePopover,+ )+ }+ >+ <div className="min-w-0 flex-1">+ <p className="truncate font-bold" title={alert.name}>+ {alert.name}+ </p>+ <p className="text-muted-foreground mt-0.5 text-xs">+ {renderEvaluatorAlertTriggerCondition(alert)}+ <span className="px-1" aria-hidden="true">+ ·+ </span>+ {renderEvaluatorAlertLastFired(alert.alertedAt)}+ </p>+ </div>+ <MonitorSeverityBadge+ severity={alert.severity}+ className="ml-auto w-auto shrink-0 px-2 py-0.5 text-[10px]"+ />+ <ArrowUpRight className="text-muted-foreground h-4 w-4 shrink-0" />+ </CommandItem>+ ))}+ </div>+ {hasMore ? (+ <CommandItem+ value="see all alerts"+ className="mt-1"+ onSelect={() =>+ navigate(+ evaluatorAlertsListUrl(+ projectId,+ props.scope === "evaluator"+ ? props.evaluatorId+ : undefined,+ ),+ closePopover,+ )+ }+ >+ <span>See all alerts</span>+ <ArrowUpRight className="text-muted-foreground ml-auto h-4 w-4 shrink-0" />+ </CommandItem>+ ) : null}+ </div>+ ) : (+ <div className="px-4 py-3">+ <p className="text-sm font-bold">+ {isAggregateCost+ ? "No alerts on evaluator cost"+ : "No alerts on this evaluator"}+ </p>+ <p className="text-muted-foreground mt-0.5 text-xs">+ {isAggregateCost+ ? "Get notified when evaluator cost crosses a threshold."+ : supportsCostAlert+ ? "Get notified when this evaluator's scores or cost cross a threshold."+ : "Get notified when this evaluator's score crosses a threshold."}+ </p>+ </div>+ )}+ <CommandSeparator className="mx-0" />+ {connectedAlerts.length > 0 ? (+ <div className="flex items-center justify-between gap-3 p-2">+ <span className="text-muted-foreground flex h-8 items-center px-1 text-sm leading-none">+ Add alert+ </span>+ <div className="flex flex-wrap items-center justify-end gap-1.5">+ {selectedScoreDataType ? (+ <Button+ type="button"+ variant="outline"+ size="sm"+ className="h-8 cursor-pointer gap-1.5"+ disabled={createDisabled}+ onClick={() =>+ openAlertCreation(+ "score",+ scoreAlertUrl(selectedScoreDataType),+ closePopover,+ selectedScoreDataType,+ )+ }+ >+ <Gauge className="text-muted-foreground h-4 w-4" />+ Score+ </Button>+ ) : null}+ {supportsCostAlert ? (+ <Button+ type="button"+ variant="outline"+ size="sm"+ className="h-8 cursor-pointer gap-1.5"+ disabled={createDisabled}+ onClick={() =>+ openAlertCreation(+ isAggregateCost ? "all_evaluator_cost" : "cost",+ costAlertUrl,+ closePopover,+ )+ }+ >+ <DollarSign className="text-muted-foreground h-4 w-4" />+ Cost+ </Button>+ ) : null}+ </div>+ </div>+ ) : (+ <CommandGroup>+ {selectedScoreDataType ? (+ <CommandItem+ disabled={createDisabled}+ onSelect={() =>+ openAlertCreation(+ "score",+ scoreAlertUrl(selectedScoreDataType),+ closePopover,+ selectedScoreDataType,+ )+ }+ >+ <Gauge className="text-muted-foreground" />+ <div>+ <p>Score threshold</p>+ <p className="text-muted-foreground text-xs">+ Alert when this evaluator's score crosses a+ threshold+ </p>+ </div>+ <ArrowUpRight className="text-muted-foreground ml-auto h-4 w-4 shrink-0" />+ </CommandItem>+ ) : null}+ {supportsCostAlert ? (+ <CommandItem+ disabled={createDisabled}+ onSelect={() =>+ openAlertCreation(+ isAggregateCost ? "all_evaluator_cost" : "cost",+ costAlertUrl,+ closePopover,+ )+ }+ >+ <DollarSign className="text-muted-foreground" />+ <div>+ <p>Cost threshold</p>+ <p className="text-muted-foreground text-xs">+ {isAggregateCost+ ? "Alert when evaluator cost crosses a threshold"+ : "Alert on spend from running this evaluator"}+ </p>+ </div>+ <ArrowUpRight className="text-muted-foreground ml-auto h-4 w-4 shrink-0" />+ </CommandItem>+ ) : null}+ </CommandGroup>+ )}+ {createDisabled ? (+ <p className="text-muted-foreground px-3 pb-2 text-xs">+ {disabledReason}+ </p>+ ) : null}+ </CommandList>+ </Command>+ )}+ >+ {({ Trigger }) => (+ <Trigger asChild>+ <Button+ type="button"+ variant="outline"+ aria-label={+ isLoading+ ? "Loading evaluator alerts"+ : alertCount > 0+ ? `${alertCount} connected evaluator alert${alertCount === 1 ? "" : "s"}`+ : "Add evaluator alert"+ }+ className="w-auto max-w-full justify-start"+ >+ {isLoading ? (+ <LoaderCircle+ className="mr-1 h-4 w-4 shrink-0 animate-spin"+ aria-hidden="true"+ />+ ) : alertCount > 0 ? (+ <Bell className="mr-1 h-4 w-4 shrink-0" aria-hidden="true" />+ ) : (+ <Plus className="mr-1 h-4 w-4 shrink-0" aria-hidden="true" />+ )}+ <span className="flex-1 text-left">+ {alertCount > 0 ? "Alerts" : "Add alert"}+ </span>+ {!isLoading && alertCount > 0 ? (+ <Badge variant="secondary" size="sm" className="ml-1 shrink-0">+ {alertCount}+ </Badge>+ ) : null}+ <ChevronDown className="h-4 w-4 shrink-0 opacity-50" />+ </Button>+ </Trigger>+ )}+ </PopoverController>+ );+}diff --git a/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker.tsx b/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker.tsxindex c1ac378533..04e4e138b9 100644--- a/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker.tsx+++ b/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker.tsx@@ -68,6 +68,7 @@ type JudgeModelPickerTriggerProps = Omit<selectedModel: JudgeModel | null;disabled: boolean;missingDefaultLabel?: string;+ borderVariant?: "default" | "contrast";};export const JudgeModelPickerTrigger = forwardRef<@@ -86,6 +87,7 @@ export const JudgeModelPickerTrigger = forwardRef<loading,loadingText,disabled,+ borderVariant = "default",...buttonProps},forwardedRef,@@ -114,6 +116,7 @@ export const JudgeModelPickerTrigger = forwardRef<className={cn(selectTriggerClassName,"w-auto max-w-full min-w-0 justify-start",+ borderVariant === "contrast" && "border-border-contrast",)}>{mode === "default" ? (diff --git a/web/src/features/evals/v2/components/VariableMapping/VariableMapping.tsx b/web/src/features/evals/v2/components/VariableMapping/VariableMapping.tsxindex d5e925db16..f4e2451b12 100644--- a/web/src/features/evals/v2/components/VariableMapping/VariableMapping.tsx+++ b/web/src/features/evals/v2/components/VariableMapping/VariableMapping.tsx@@ -52,7 +52,7 @@ function ReadOnlyVariableMapping({function EmptyVariableMapping({ mode }: Pick<VariableMappingProps, "mode">) {return (- <div className="border-border bg-muted/20 flex flex-col items-center justify-center gap-2 rounded-md border border-dashed px-6 py-8 text-center">+ <div className="border-border bg-muted/20 flex flex-col items-center justify-center gap-2 rounded-md border px-6 py-8 text-center"><div className="bg-muted text-muted-foreground flex size-9 items-center justify-center rounded-full"><Braces className="size-4" aria-hidden /></div>diff --git a/web/src/features/evals/v2/fns/evaluators/evaluatorAlertUrl.clienttest.ts b/web/src/features/evals/v2/fns/evaluators/evaluatorAlertUrl.clienttest.tsnew file mode 100644index 0000000000..d4bc41698e--- /dev/null+++ b/web/src/features/evals/v2/fns/evaluators/evaluatorAlertUrl.clienttest.ts@@ -0,0 +1,48 @@+import { describe, expect, it } from "vitest";++import {+ evaluatorAlertsListUrl,+ evaluatorAlertUrl,+} from "@/src/features/evals/v2/fns/evaluators/evaluatorAlertUrl";++describe("evaluatorAlertUrl", () => {+ it("builds evaluator-specific score and cost alert URLs", () => {+ expect(+ evaluatorAlertUrl("project/1", {+ type: "score",+ evaluatorId: "eval/1",+ scoreDataType: "BOOLEAN",+ }),+ ).toBe(+ "/project/project%2F1/alerts/new?alert=evaluatorScore&evaluatorId=eval%2F1&scoreDataType=BOOLEAN",+ );+ expect(+ evaluatorAlertUrl("project/1", {+ type: "cost",+ evaluatorId: "eval/1",+ }),+ ).toBe(+ "/project/project%2F1/alerts/new?alert=evaluatorCost&evaluatorId=eval%2F1",+ );+ });++ it("builds the all-evaluator spend URL", () => {+ expect(evaluatorAlertUrl("project/1", { type: "allEvaluatorCost" })).toBe(+ "/project/project%2F1/alerts/new?alert=allEvaluatorCost",+ );+ });+});++describe("evaluatorAlertsListUrl", () => {+ it("filters evaluator-specific alerts by evaluator ID", () => {+ expect(evaluatorAlertsListUrl("project/1", "eval/1")).toBe(+ "/project/project%2F1/alerts?filter=evaluatorId%3BstringOptions%3B%3Bany+of%3Beval%252F1",+ );+ });++ it("leaves the all-evaluator alerts list unfiltered", () => {+ expect(evaluatorAlertsListUrl("project/1")).toBe(+ "/project/project%2F1/alerts",+ );+ });+});diff --git a/web/src/features/evals/v2/fns/evaluators/evaluatorAlertUrl.ts b/web/src/features/evals/v2/fns/evaluators/evaluatorAlertUrl.tsnew file mode 100644index 0000000000..7d046843b7--- /dev/null+++ b/web/src/features/evals/v2/fns/evaluators/evaluatorAlertUrl.ts@@ -0,0 +1,50 @@+import { encodeFiltersGeneric } from "@langfuse/shared";++/** Builds an alerts-list URL, optionally filtered to one evaluator. */+export function evaluatorAlertsListUrl(+ projectId: string,+ evaluatorId?: string,+): string {+ const base = `/project/${encodeURIComponent(projectId)}/alerts`;+ if (!evaluatorId) return base;++ const filter = encodeFiltersGeneric([+ {+ column: "evaluatorId",+ type: "stringOptions",+ operator: "any of",+ value: [evaluatorId],+ },+ ]);+ const params = new URLSearchParams({ filter });+ return `${base}?${params.toString()}`;+}++/** Builds a new-alert URL using only evaluator alert prefill fields. */+export function evaluatorAlertUrl(+ projectId: string,+ alert:+ | {+ type: "score";+ evaluatorId: string;+ scoreDataType: "NUMERIC" | "BOOLEAN" | "CATEGORICAL";+ }+ | { type: "cost"; evaluatorId: string }+ | { type: "allEvaluatorCost" },+): string {+ const base = `/project/${encodeURIComponent(projectId)}/alerts/new`;+ const params = new URLSearchParams();++ if (alert.type === "allEvaluatorCost") {+ params.set("alert", "allEvaluatorCost");+ } else if (alert.type === "cost") {+ params.set("alert", "evaluatorCost");+ params.set("evaluatorId", alert.evaluatorId);+ } else {+ params.set("alert", "evaluatorScore");+ params.set("evaluatorId", alert.evaluatorId);+ params.set("scoreDataType", alert.scoreDataType);+ }++ return `${base}?${params.toString()}`;+}diff --git a/web/src/features/evals/v2/fns/evaluators/getFirstCodeEvaluatorScoreDataType.clienttest.ts b/web/src/features/evals/v2/fns/evaluators/getFirstCodeEvaluatorScoreDataType.clienttest.tsnew file mode 100644index 0000000000..5bee85f703--- /dev/null+++ b/web/src/features/evals/v2/fns/evaluators/getFirstCodeEvaluatorScoreDataType.clienttest.ts@@ -0,0 +1,41 @@+import { getFirstCodeEvaluatorScoreDataType } from "./getFirstCodeEvaluatorScoreDataType";++describe("getFirstCodeEvaluatorScoreDataType", () => {+ it("uses the first supported score data type in TypeScript source", () => {+ expect(+ getFirstCodeEvaluatorScoreDataType(`+ return {+ scores: [+ { name: "Passed", dataType: "BOOLEAN", value: true },+ { name: "Confidence", dataType: "NUMERIC", value: 1 },+ ],+ };+ `),+ ).toBe("BOOLEAN");+ });++ it("reads quoted Python dictionary keys", () => {+ expect(+ getFirstCodeEvaluatorScoreDataType(`+ return {+ "scores": [+ {"name": "Confidence", "dataType": "NUMERIC", "value": 1}+ ]+ }+ `),+ ).toBe("NUMERIC");+ });++ it("does not skip an unsupported first score type", () => {+ expect(+ getFirstCodeEvaluatorScoreDataType(`+ return {+ scores: [+ { name: "Explanation", dataType: "TEXT", value: "ok" },+ { name: "Confidence", dataType: "NUMERIC", value: 1 },+ ],+ };+ `),+ ).toBeUndefined();+ });+});diff --git a/web/src/features/evals/v2/fns/evaluators/getFirstCodeEvaluatorScoreDataType.ts b/web/src/features/evals/v2/fns/evaluators/getFirstCodeEvaluatorScoreDataType.tsnew file mode 100644index 0000000000..b24460feef--- /dev/null+++ b/web/src/features/evals/v2/fns/evaluators/getFirstCodeEvaluatorScoreDataType.ts@@ -0,0 +1,21 @@+export type AlertScoreDataType = "NUMERIC" | "BOOLEAN" | "CATEGORICAL";++const ALERT_SCORE_DATA_TYPES = new Set<AlertScoreDataType>([+ "NUMERIC",+ "BOOLEAN",+ "CATEGORICAL",+]);++/** Reads the first score data type declared in stored code evaluator source. */+export function getFirstCodeEvaluatorScoreDataType(+ sourceCode: string,+): AlertScoreDataType | undefined {+ const match = sourceCode.match(+ /(?:["']dataType["']|dataType)\s*:\s*(?:["'](NUMERIC|BOOLEAN|CATEGORICAL|TEXT)["']|ScoreDataTypeEnum\.(NUMERIC|BOOLEAN|CATEGORICAL|TEXT))/,+ );+ const dataType = match?.[1] ?? match?.[2];++ return dataType && ALERT_SCORE_DATA_TYPES.has(dataType as AlertScoreDataType)+ ? (dataType as AlertScoreDataType)+ : undefined;+}diff --git a/web/src/features/evals/v2/fns/evaluators/renderEvaluatorAlertLastFired.clienttest.ts b/web/src/features/evals/v2/fns/evaluators/renderEvaluatorAlertLastFired.clienttest.tsnew file mode 100644index 0000000000..83efde8fab--- /dev/null+++ b/web/src/features/evals/v2/fns/evaluators/renderEvaluatorAlertLastFired.clienttest.ts@@ -0,0 +1,24 @@+// @vitest-environment node++import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";++import { renderEvaluatorAlertLastFired } from "./renderEvaluatorAlertLastFired";++describe("renderEvaluatorAlertLastFired", () => {+ beforeEach(() => {+ vi.useFakeTimers();+ vi.setSystemTime(new Date("2026-06-03T12:00:00.000Z"));+ });++ afterEach(() => vi.useRealTimers());++ it("renders the relative time of the latest alert emission", () => {+ expect(+ renderEvaluatorAlertLastFired(new Date("2026-06-01T12:00:00.000Z")),+ ).toBe("Last fired 2 days ago");+ });++ it("renders alerts that have not fired", () => {+ expect(renderEvaluatorAlertLastFired(null)).toBe("Never fired");+ });+});diff --git a/web/src/features/evals/v2/fns/evaluators/renderEvaluatorAlertLastFired.ts b/web/src/features/evals/v2/fns/evaluators/renderEvaluatorAlertLastFired.tsnew file mode 100644index 0000000000..b6d4940a64--- /dev/null+++ b/web/src/features/evals/v2/fns/evaluators/renderEvaluatorAlertLastFired.ts@@ -0,0 +1,8 @@+import { formatDistanceToNowStrict } from "date-fns";++/** Renders the most recent evaluator alert emission time. */+export function renderEvaluatorAlertLastFired(alertedAt: Date | null): string {+ return alertedAt+ ? `Last fired ${formatDistanceToNowStrict(alertedAt, { addSuffix: true })}`+ : "Never fired";+}diff --git a/web/src/features/evals/v2/fns/evaluators/renderEvaluatorAlertTriggerCondition.clienttest.ts b/web/src/features/evals/v2/fns/evaluators/renderEvaluatorAlertTriggerCondition.clienttest.tsnew file mode 100644index 0000000000..5830b21738--- /dev/null+++ b/web/src/features/evals/v2/fns/evaluators/renderEvaluatorAlertTriggerCondition.clienttest.ts@@ -0,0 +1,27 @@+// @vitest-environment node++import { describe, expect, it } from "vitest";++import { renderEvaluatorAlertTriggerCondition } from "./renderEvaluatorAlertTriggerCondition";++describe("renderEvaluatorAlertTriggerCondition", () => {+ it("renders a count threshold", () => {+ expect(+ renderEvaluatorAlertTriggerCondition({+ metric: { measure: "count", aggregation: "count" },+ thresholdOperator: "GT",+ alertThreshold: 5,+ }),+ ).toBe("Count > 5");+ });++ it("renders an aggregated measure threshold", () => {+ expect(+ renderEvaluatorAlertTriggerCondition({+ metric: { measure: "value", aggregation: "avg" },+ thresholdOperator: "LTE",+ alertThreshold: 0.5,+ }),+ ).toBe("Avg value ≤ 0.5");+ });+});diff --git a/web/src/features/evals/v2/fns/evaluators/renderEvaluatorAlertTriggerCondition.ts b/web/src/features/evals/v2/fns/evaluators/renderEvaluatorAlertTriggerCondition.tsnew file mode 100644index 0000000000..63c3dcca8b--- /dev/null+++ b/web/src/features/evals/v2/fns/evaluators/renderEvaluatorAlertTriggerCondition.ts@@ -0,0 +1,37 @@+import { startCase } from "lodash";+import type { z } from "zod";++import { type metricAggregations } from "@langfuse/shared";+import type { MonitorThresholdOperator } from "@langfuse/shared/monitors";++const operatorSymbol: Record<MonitorThresholdOperator, string> = {+ GT: ">",+ GTE: "≥",+ LT: "<",+ LTE: "≤",+ EQ: "=",+ NEQ: "≠",+};++const aggregationLabel = (+ aggregation: z.infer<typeof metricAggregations>,+): string =>+ /^p\d+$/.test(aggregation) ? aggregation : startCase(aggregation);++/** Renders an evaluator alert boundary, e.g. "Count > 5". */+export function renderEvaluatorAlertTriggerCondition({+ metric,+ thresholdOperator,+ alertThreshold,+}: {+ metric: { measure: string; aggregation: z.infer<typeof metricAggregations> };+ thresholdOperator: MonitorThresholdOperator;+ alertThreshold: number;+}): string {+ const metricLabel =+ metric.measure === "count"+ ? "Count"+ : `${aggregationLabel(metric.aggregation)} ${startCase(metric.measure).toLowerCase()}`;++ return `${metricLabel} ${operatorSymbol[thresholdOperator]} ${alertThreshold}`;+}diff --git a/web/src/features/evals/v2/hooks/useEvaluatorAlerts.ts b/web/src/features/evals/v2/hooks/useEvaluatorAlerts.tsnew file mode 100644index 0000000000..1cbc865778--- /dev/null+++ b/web/src/features/evals/v2/hooks/useEvaluatorAlerts.ts@@ -0,0 +1,56 @@+import { useEntitlementLimit } from "@/src/features/entitlements";+import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";+import { api } from "@/src/utils/api";++/** Owns evaluator alert permissions, linked-alert loading, and creation limits. */+export function useEvaluatorAlerts(+ params:+ | { scope: "evaluator"; projectId: string; evaluatorId: string | null }+ | { scope: "allEvaluators"; projectId: string },+) {+ const canRead = useHasProjectAccess({+ projectId: params.projectId,+ scope: "alerts:read",+ });+ const canCreate = useHasProjectAccess({+ projectId: params.projectId,+ scope: "alerts:CUD",+ });+ const monitorEntitlementLimit = useEntitlementLimit("monitor-count");+ const evaluatorId =+ params.scope === "evaluator" ? (params.evaluatorId ?? "") : "";++ const evaluatorAlerts = api.monitors.linkedEvaluatorAlerts.useQuery(+ { projectId: params.projectId, evaluatorId },+ {+ enabled: Boolean(params.scope === "evaluator" && evaluatorId && canRead),+ },+ );+ const aggregateCostAlerts =+ api.monitors.linkedAllEvaluatorSpendAlerts.useQuery(+ { projectId: params.projectId },+ {+ enabled: Boolean(+ params.scope === "allEvaluators" && params.projectId && canRead,+ ),+ },+ );+ const monitorCount = api.monitors.count.useQuery(+ { projectId: params.projectId },+ { enabled: Boolean(params.projectId && canCreate) },+ );+ const limitReached =+ typeof monitorEntitlementLimit === "number" &&+ (monitorCount.data?.count ?? 0) >= monitorEntitlementLimit;+ const linkedAlerts =+ params.scope === "evaluator" ? evaluatorAlerts : aggregateCostAlerts;++ return {+ connectedAlerts: linkedAlerts.data?.data ?? [],+ hasMore: linkedAlerts.data?.hasMore ?? false,+ isLoading: canRead && linkedAlerts.isPending,+ canRead,+ canCreate,+ limitReached,+ };+}diff --git a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsxindex 60785fbbf9..1f66f216c3 100644--- a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx+++ b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx@@ -51,7 +51,11 @@ import { EvaluatorBlockedBanner } from "@/src/features/evals/v2/components/Evaluimport { useIsMobile } from "@/src/hooks/use-mobile";import { prepareEvaluatorMetadataForSave } from "@/src/features/evals/v2/fns/prepareEvaluatorMetadataForSave";import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";+import { useEvaluatorAlerts } from "@/src/features/evals/v2/hooks/useEvaluatorAlerts";import { useCodeEvalSourceValidation } from "@/src/features/evals/hooks/useCodeEvalSourceValidation";+import { EvaluatorAlertButton } from "@/src/features/evals/v2/components/Evaluators/EvaluatorAlertButton/EvaluatorAlertButton";+import { toScoreOutputFormState } from "@/src/features/evals/v2/fns/scoreOutput/toScoreOutputFormState";+import { getFirstCodeEvaluatorScoreDataType } from "@/src/features/evals/v2/fns/evaluators/getFirstCodeEvaluatorScoreDataType";import {getEvaluatorCreationAnalyticsProperties,getJudgePromptAnalyticsProperties,@@ -164,6 +168,19 @@ export function EvaluatorSetupPage(projectId,scope: "evaluator:CUD",});+ const evaluatorAlerts = useEvaluatorAlerts({+ scope: "evaluator",+ projectId,+ evaluatorId: initialEvaluator?.id ?? null,+ });+ const scoreDataType = initialEvaluator+ ? initialEvaluator.definition.type === "LLM_AS_JUDGE"+ ? toScoreOutputFormState(initialEvaluator.definition.outputDefinition)+ .dataType+ : getFirstCodeEvaluatorScoreDataType(+ initialEvaluator.definition.sourceCode,+ )+ : undefined;const projectDefaultModel = useProjectDefaultModel({projectId,source: "editor",@@ -677,6 +694,14 @@ export function EvaluatorSetupPage(initialEvaluator.definition.variableMapping}/>+ <EvaluatorAlertButton+ scope="evaluator"+ projectId={projectId}+ evaluatorId={initialEvaluator.id}+ evaluatorType={initialEvaluator.type}+ scoreDataType={scoreDataType}+ {...evaluatorAlerts}+ /><Buttontype="button"variant="outline"diff --git a/web/src/features/evals/v2/pages/EvaluatorsPage.tsx b/web/src/features/evals/v2/pages/EvaluatorsPage.tsxindex f7626a24cb..b3ef4b696b 100644--- a/web/src/features/evals/v2/pages/EvaluatorsPage.tsx+++ b/web/src/features/evals/v2/pages/EvaluatorsPage.tsx@@ -36,6 +36,7 @@ import { usePostHogClientCapture } from "@/src/features/posthog-analytics";import useColumnOrder from "@/src/features/column-visibility/hooks/useColumnOrder";import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";+import { useEvaluatorAlerts } from "@/src/features/evals/v2/hooks/useEvaluatorAlerts";import { TableSelectionManager } from "@/src/features/table/components/TableSelectionManager";import { usePaginationState } from "@/src/hooks/usePaginationState";import { useSidebarFilterState } from "@/src/features/filters/hooks/useSidebarFilterState";@@ -72,6 +73,7 @@ import { V4MigrationUpdateRequiredBadge } from "@/src/features/v4-migration/V4Miimport { createNumberTableColumn } from "@/src/components/design-system/table/columns/createNumberTableColumn";import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";import { createUserTableColumn } from "@/src/components/design-system/table/columns/createUserTableColumn";+import { EvaluatorAlertButton } from "@/src/features/evals/v2/components/Evaluators/EvaluatorAlertButton/EvaluatorAlertButton";type EvaluatorRow = RouterOutputs["evalsV2"]["list"]["evaluators"][number];@@ -170,6 +172,10 @@ export default function EvaluatorsPage() {const router = useRouter();const capture = usePostHogClientCapture();const projectId = router.query.projectId as string;+ const evaluatorAlerts = useEvaluatorAlerts({+ scope: "allEvaluators",+ projectId,+ });const projectDefaultModel = useProjectDefaultModel({projectId,source: "overview",@@ -617,10 +623,18 @@ export default function EvaluatorsPage() {projectDefaultModel.connectionsPending ||projectDefaultModel.update.isPending}+ borderVariant="contrast"/></PopoverTrigger></JudgeModelPicker>)}+ {showOnboarding ? null : (+ <EvaluatorAlertButton+ scope="allEvaluators"+ projectId={projectId}+ {...evaluatorAlerts}+ />+ )}<Button onClick={() => setGalleryOpen(true)}><Plus className="mr-2 h-4 w-4" />New evaluatordiff --git a/web/src/features/filters/components/filter-builder.tsx b/web/src/features/filters/components/filter-builder.tsxindex b8b6388138..bf0d23a0c1 100644--- a/web/src/features/filters/components/filter-builder.tsx+++ b/web/src/features/filters/components/filter-builder.tsx@@ -667,6 +667,10 @@ function FilterBuilderForm({const stringObjectSuggest =column?.type === "stringObject" &&columnsWithCustomSelect.includes(column.id);+ const stringSuggest =+ column?.type === "string" &&+ columnsWithCustomSelect.includes(column.id) &&+ column.options !== undefined;const columnLabel = column ? column.name : "Column";const columnsForPicker = getColumnOptionsForFilterRow(columns,@@ -900,8 +904,11 @@ function FilterBuilderForm({{...filter,operator: value as any,- // Ensure null filters always have empty string value- value: filter.type === "null" ? "" : (filter.value as any),+ // Value-less operators keep an empty string for schema compatibility.+ value:+ filter.type === "null" || value === "is not empty"+ ? ""+ : (filter.value as any),},i,);@@ -925,7 +932,9 @@ function FilterBuilderForm({const valueControl = keyPending ? (<Input disabled />- ) : stringObjectSuggest && filter.type === "stringObject" ? (+ ) : filter.type === "string" &&+ filter.operator === "is not empty" ? null : stringObjectSuggest &&+ filter.type === "stringObject" ? (<SingleSelecttitle="Value"className="min-w-[100px]"@@ -935,6 +944,16 @@ function FilterBuilderForm({disabled={disabled}isCustomSelectEnabled/>+ ) : stringSuggest && filter.type === "string" ? (+ <SingleSelect+ title="Value"+ className="min-w-[100px]"+ options={column?.type === "string" ? (column.options ?? []) : []}+ value={filter.value ?? undefined}+ onValueChange={(value) => handleFilterChange({ ...filter, value }, i)}+ disabled={disabled}+ isCustomSelectEnabled+ />) : filter.type === "string" || filter.type === "stringObject" ? (<Inputdisabled={disabled}diff --git a/web/src/features/filters/components/single-select.clienttest.tsx b/web/src/features/filters/components/single-select.clienttest.tsxindex ad2e6854d6..dc1c8ae86a 100644--- a/web/src/features/filters/components/single-select.clienttest.tsx+++ b/web/src/features/filters/components/single-select.clienttest.tsx@@ -54,4 +54,23 @@ describe("SingleSelect", () => {expect(onValueChange).toHaveBeenCalledWith("release");});++ it("renders display labels but commits stable option values", () => {+ const onValueChange = vi.fn();+ render(+ <SingleSelect+ title="Evaluator"+ options={[+ { value: "evaluator-1", displayValue: "Answer quality" },+ { value: "evaluator-2", displayValue: "Hallucination" },+ ]}+ onValueChange={onValueChange}+ />,+ );++ open();+ fireEvent.click(screen.getByRole("option", { name: "Answer quality" }));++ expect(onValueChange).toHaveBeenCalledWith("evaluator-1");+ });});diff --git a/web/src/features/filters/config/monitors-config.ts b/web/src/features/filters/config/monitors-config.tsindex 845377065f..0fcdf477e3 100644--- a/web/src/features/filters/config/monitors-config.ts+++ b/web/src/features/filters/config/monitors-config.ts@@ -1,26 +1,45 @@import { monitorsTableCols } from "@langfuse/shared";import type { FilterConfig } from "@/src/features/filters/lib/filter-config";+const baseFacets: FilterConfig["facets"] = [+ {+ type: "categorical" as const,+ column: "severity",+ label: "Severity",+ disableTextFilter: true,+ },+ {+ type: "categorical" as const,+ column: "tags",+ label: "Tags",+ },+];+export const monitorFilterConfig: FilterConfig = {tableName: "monitors",-columnDefinitions: monitorsTableCols,-defaultExpanded: ["severity", "tags"],-defaultSidebarCollapsed: false,-- facets: [- {- type: "categorical" as const,- column: "severity",- label: "Severity",- disableTextFilter: true,- },- {- type: "categorical" as const,- column: "tags",- label: "Tags",- },- ],+ facets: baseFacets,};++/** Adds the evaluator facet only when the project has evaluator-specific alerts. */+export const getMonitorFilterConfig = (+ hasEvaluatorOptions: boolean,+): FilterConfig =>+ hasEvaluatorOptions+ ? {+ ...monitorFilterConfig,+ defaultExpanded: ["severity", "evaluatorId", "tags"],+ facets: [+ baseFacets[0],+ {+ type: "categorical" as const,+ column: "evaluatorId",+ label: "Evaluator",+ disableTextFilter: true,+ },+ baseFacets[1],+ ],+ }+ : monitorFilterConfig;diff --git a/web/src/features/metrics/components/MetricsFilterBuilder.clienttest.ts b/web/src/features/metrics/components/MetricsFilterBuilder.clienttest.tsindex 88cbcdc9a5..516821c6df 100644--- a/web/src/features/metrics/components/MetricsFilterBuilder.clienttest.ts+++ b/web/src/features/metrics/components/MetricsFilterBuilder.clienttest.ts@@ -2,7 +2,7 @@import { describe, expect, it } from "vitest";-import { type FilterState } from "@langfuse/shared";+import { filterOperators, type FilterState } from "@langfuse/shared";import {getMetricsColumnsWithCustomSelect,@@ -54,6 +54,44 @@ describe("editorFiltersToViewFilters", () => {),).toEqual(filters);});++ it("canonicalizes current and legacy evaluator display labels", () => {+ const canonical: FilterState = [+ {+ column: "evaluatorId",+ type: "string",+ operator: "=",+ value: "evaluator-1",+ },+ ];+ for (const column of ["Evaluator", "Evaluator ID"]) {+ expect(+ editorFiltersToViewFilters("observations", [+ {+ ...canonical[0],+ column,+ },+ ]),+ ).toEqual(canonical);+ }+ expect(+ editorFiltersToViewFilters("observations", [+ {+ column: "Evaluator execution",+ type: "boolean",+ operator: "=",+ value: false,+ },+ ]),+ ).toEqual([+ {+ column: "isEvaluatorTest",+ type: "boolean",+ operator: "=",+ value: false,+ },+ ]);+ });});describe("resolvesToColumn", () => {@@ -91,6 +129,23 @@ describe("resolvesToColumn", () => {expect(resolves("isRootObservation", "observations", "v1")).toBe(false);});+ it("offers an is-not-empty operator for evaluator IDs", () => {+ const evaluatorColumn = columnsFor("observations").find(+ (column) => column.id === "evaluatorId",+ );++ expect(evaluatorColumn?.type).toBe("string");+ expect(filterOperators.string).toContain("is not empty");+ });++ it("renders evaluator alert prefill filters", () => {+ expect(resolves("evaluatorId", "observations", "v1")).toBe(true);+ expect(resolves("evaluatorId", "observations", "v2")).toBe(true);+ expect(resolves("isEvaluatorTest", "observations", "v2")).toBe(true);+ expect(resolves("evaluatorId", "scores-categorical", "v2")).toBe(true);+ expect(resolves("isEvaluatorTest", "scores-categorical", "v2")).toBe(true);+ });+it("unmapped column: does not resolve", () => {expect(resolves("totallyUnknownColumn", "observations")).toBe(false);});@@ -216,6 +271,27 @@ describe("buildV2FilterColumnsParams", () => {expect(getMetricsColumnsWithCustomSelect(params)).toContain("experimentId");});+ it("labels evaluator ID suggestions by name while retaining ID values", () => {+ const params = buildV2FilterColumnsParams({+ view: "observations",+ filterOptions: undefined,+ datasets: undefined,+ evaluatorOptions: [+ { value: "evaluator-1", displayValue: "Answer quality" },+ ],+ });+ const column = getMetricsFilterColumns(params).find(+ (candidate) => candidate.id === "evaluatorId",+ );++ expect(column?.name).toBe("Evaluator");+ expect(column?.type).toBe("string");+ expect(column && "options" in column ? column.options : []).toEqual([+ { value: "evaluator-1", displayValue: "Answer quality" },+ ]);+ expect(getMetricsColumnsWithCustomSelect(params)).toContain("evaluatorId");+ });+it("wires metadata key suggestions into the Metadata column", () => {const params = buildV2FilterColumnsParams({view: "observations",diff --git a/web/src/features/metrics/components/MetricsFilterBuilder.tsx b/web/src/features/metrics/components/MetricsFilterBuilder.tsxindex 698499862a..af0986a347 100644--- a/web/src/features/metrics/components/MetricsFilterBuilder.tsx+++ b/web/src/features/metrics/components/MetricsFilterBuilder.tsx@@ -91,8 +91,30 @@ export const MetricsFilterBuilder = ({version,...props}: MetricsFilterFetcherProps & { version: ViewVersion }) => {- if (version === "v1") return <MetricsFilterBuilderV1 {...props} />;- return <MetricsFilterBuilderV2 {...props} />;+ const evaluatorOptions = api.evalsV2.options.useQuery(+ { projectId: props.projectId, limit: 100 },+ v2FilterOptionsQueryConfig,+ );+ const evaluatorNameOptions =+ evaluatorOptions.data?.map(({ id, name }) => ({+ value: id,+ displayValue: name,+ })) ?? [];++ if (version === "v1") {+ return (+ <MetricsFilterBuilderV1+ {...props}+ evaluatorOptions={evaluatorNameOptions}+ />+ );+ }+ return (+ <MetricsFilterBuilderV2+ {...props}+ evaluatorOptions={evaluatorNameOptions}+ />+ );};/** MetricsFilterDateRange is the preview/lookback window used to scope filter-value discovery. */@@ -107,6 +129,11 @@ type MetricsFilterFetcherProps = {onChange: (filters: FilterState) => void;};+type MetricsFilterFetcherWithEvaluatorOptionsProps =+ MetricsFilterFetcherProps & {+ evaluatorOptions: SingleValueOption[];+ };+/** MetricsFilterBuilderV1 loads the v1 (traces + generations + project) filter options and renders the filter view. */const MetricsFilterBuilderV1 = ({view,@@ -114,7 +141,8 @@ const MetricsFilterBuilderV1 = ({dateRange,filters,onChange,-}: MetricsFilterFetcherProps) => {+ evaluatorOptions,+}: MetricsFilterFetcherWithEvaluatorOptionsProps) => {const traceFilterOptions = api.traces.filterOptions.useQuery({projectId,@@ -143,6 +171,7 @@ const MetricsFilterBuilderV1 = ({traceFilterOptions: traceFilterOptions.data,generationsFilterOptions: generationsFilterOptions.data,environmentFilterOptions: environmentFilterOptions.data,+ evaluatorOptions,});return (@@ -163,7 +192,8 @@ const MetricsFilterBuilderV2 = ({dateRange,filters,onChange,-}: MetricsFilterFetcherProps) => {+ evaluatorOptions,+}: MetricsFilterFetcherWithEvaluatorOptionsProps) => {const startTimeFilter = metricsFilterTimeFilter("startTime", dateRange);const eventsFilterOptions = api.events.filterOptions.useQuery(@@ -187,6 +217,7 @@ const MetricsFilterBuilderV2 = ({filterOptions: eventsFilterOptions.data,slowFilterOptions: slowEventsFilterOptions.data,datasets: datasets.data,+ evaluatorOptions,metadataKeys: eventsFilterOptions.data?.metadataKeys?.map((row) => row.value,),@@ -286,6 +317,7 @@ const buildV1FilterColumnsParams = ({traceFilterOptions,generationsFilterOptions,environmentFilterOptions,+ evaluatorOptions = [],}: {view: MetricsFilterFetcherProps["view"];traceFilterOptions: RouterOutputs["traces"]["filterOptions"] | undefined;@@ -295,6 +327,7 @@ const buildV1FilterColumnsParams = ({environmentFilterOptions:| RouterOutputs["projects"]["environmentFilterOptions"]| undefined;+ evaluatorOptions?: SingleValueOption[];}): GetMetricsFilterColumnsParams => ({selectedView: view,viewVersion: "v1",@@ -320,6 +353,7 @@ const buildV1FilterColumnsParams = ({releaseOptions: [],scoreNameOptions: [],experimentIdOptions: [],+ evaluatorOptions,metadataKeyOptions: [],});@@ -329,12 +363,14 @@ const buildV2FilterColumnsParams = ({filterOptions,slowFilterOptions,datasets,+ evaluatorOptions = [],metadataKeys,}: {view: z.infer<typeof views>;filterOptions: RouterOutputs["events"]["filterOptions"] | undefined;slowFilterOptions?: RouterOutputs["events"]["filterOptions"];datasets: Array<{ id: string; name: string }> | undefined;+ evaluatorOptions?: SingleValueOption[];metadataKeys?: string[];}): GetMetricsFilterColumnsParams => {const datasetIds = new Set(@@ -365,6 +401,7 @@ const buildV2FilterColumnsParams = ({experimentIdOptions: normalizeSingleValueOptions(filterOptions?.experimentId,),+ evaluatorOptions,metadataKeyOptions: metadataKeys ?? [],};};diff --git a/web/src/features/metrics/metricsFilterColumns.ts b/web/src/features/metrics/metricsFilterColumns.tsindex 93fa508a03..b9a6d4ad0e 100644--- a/web/src/features/metrics/metricsFilterColumns.ts+++ b/web/src/features/metrics/metricsFilterColumns.ts@@ -27,6 +27,7 @@ export type GetMetricsFilterColumnsParams = {releaseOptions: SingleValueOption[];scoreNameOptions: SingleValueOption[];experimentIdOptions: SingleValueOption[];+ evaluatorOptions: SingleValueOption[];metadataKeyOptions: string[];};@@ -55,6 +56,7 @@ const getMetricsFilterColumnSpecs = ({releaseOptions,scoreNameOptions,experimentIdOptions,+ evaluatorOptions,metadataKeyOptions,}: GetMetricsFilterColumnsParams): MetricsFilterColumnSpec[] => {// Value suggestions come from the v2 events filter-options; v1 keeps manual@@ -158,16 +160,36 @@ const getMetricsFilterColumnSpecs = ({// "Observation Name" on the observations view filters observations.name// (mapped through dashboardUiTableToViewMapping). "Score Name" is omitted// here because observations:scores is 1:n -- see LFE-9773.- filterColumns.push({- column: {- name: "Observation Name",- id: "observationName",- type: "stringOptions",- options: observationNameOptions,- internal: "internalValue",+ filterColumns.push(+ {+ column: {+ name: "Observation Name",+ id: "observationName",+ type: "stringOptions",+ options: observationNameOptions,+ internal: "internalValue",+ },+ customSelect: true,},- customSelect: true,- });+ {+ column: {+ name: "Evaluator",+ id: "evaluatorId",+ type: "string",+ internal: "internalValue",+ options: evaluatorOptions,+ },+ customSelect: true,+ },+ {+ column: {+ name: "Evaluator test run",+ id: "isEvaluatorTest",+ type: "boolean",+ internal: "internalValue",+ },+ },+ );}if (@@ -187,6 +209,24 @@ const getMetricsFilterColumnSpecs = ({},customSelect: true,},+ {+ column: {+ name: "Evaluator",+ id: "evaluatorId",+ type: "string",+ internal: "internalValue",+ options: evaluatorOptions,+ },+ customSelect: true,+ },+ {+ column: {+ name: "Evaluator test run",+ id: "isEvaluatorTest",+ type: "boolean",+ internal: "internalValue",+ },+ },);}diff --git a/web/src/features/monitors/components/DeleteMonitorButton.clienttest.tsx b/web/src/features/monitors/components/DeleteMonitorButton.clienttest.tsxnew file mode 100644index 0000000000..ca19801d30--- /dev/null+++ b/web/src/features/monitors/components/DeleteMonitorButton.clienttest.tsx@@ -0,0 +1,62 @@+import {+ fireEvent,+ render,+ screen,+ waitFor,+ within,+} from "@testing-library/react";++import { invalidateMonitorQueriesAfterDelete } from "@/src/features/monitors/fns/invalidateMonitorQueriesAfterDelete";+import { DeleteMonitorButton } from "./DeleteMonitorButton";++describe("DeleteMonitorButton", () => {+ it("invalidates monitor lists without refetching the deleted monitor", async () => {+ const monitors = {+ all: { invalidate: vi.fn().mockResolvedValue(undefined) },+ getFilterOptions: { invalidate: vi.fn().mockResolvedValue(undefined) },+ linkedEvaluatorAlerts: {+ invalidate: vi.fn().mockResolvedValue(undefined),+ },+ get: { invalidate: vi.fn().mockResolvedValue(undefined) },+ invalidate: vi.fn().mockResolvedValue(undefined),+ };++ await invalidateMonitorQueriesAfterDelete(monitors);++ expect(monitors.all.invalidate).toHaveBeenCalledOnce();+ expect(monitors.getFilterOptions.invalidate).toHaveBeenCalledOnce();+ expect(monitors.linkedEvaluatorAlerts.invalidate).toHaveBeenCalledOnce();+ expect(monitors.get.invalidate).not.toHaveBeenCalled();+ expect(monitors.invalidate).not.toHaveBeenCalled();+ });++ it("confirms before deleting an alert", async () => {+ const onDelete = vi.fn().mockResolvedValue(undefined);+ render(+ <DeleteMonitorButton+ monitorName="Evaluator cost alert"+ deleting={false}+ onDelete={onDelete}+ />,+ );++ const trigger = screen.getByRole("button", { name: "Delete alert" });+ expect(trigger.querySelector(".text-destructive")).toBeInTheDocument();++ fireEvent.click(trigger);++ const dialog = screen.getByRole("dialog");+ expect(+ within(dialog).getByRole("heading", { name: "Delete alert" }),+ ).toBeInTheDocument();+ expect(+ within(dialog).getByText(/permanently deletes "Evaluator cost alert"/),+ ).toBeInTheDocument();++ fireEvent.click(+ within(dialog).getByRole("button", { name: "Delete alert" }),+ );++ await waitFor(() => expect(onDelete).toHaveBeenCalledOnce());+ });+});diff --git a/web/src/features/monitors/components/DeleteMonitorButton.tsx b/web/src/features/monitors/components/DeleteMonitorButton.tsxnew file mode 100644index 0000000000..97c929cbed--- /dev/null+++ b/web/src/features/monitors/components/DeleteMonitorButton.tsx@@ -0,0 +1,50 @@+import { useState } from "react";+import { Trash2 } from "lucide-react";++import { Button } from "@/src/components/ui/button";+import { ConfirmDialog } from "@/src/components/ui/confirm-dialog";++export function DeleteMonitorButton({+ monitorName,+ deleting,+ onDelete,+}: {+ monitorName: string;+ deleting: boolean;+ onDelete: () => Promise<void>;+}) {+ const [open, setOpen] = useState(false);++ const handleDelete = async () => {+ try {+ await onDelete();+ setOpen(false);+ } catch {+ // The mutation reports the error while the dialog remains open for retry.+ }+ };++ return (+ <ConfirmDialog+ open={open}+ onOpenChange={setOpen}+ title="Delete alert"+ description={`This permanently deletes "${monitorName}". This action cannot be undone.`}+ confirmLabel="Delete alert"+ loading={deleting}+ loadingText="Deleting alert..."+ onConfirm={handleDelete}+ trigger={+ <Button+ type="button"+ variant="outline"+ size="icon"+ title="Delete alert"+ aria-label="Delete alert"+ >+ <Trash2 className="text-destructive h-4 w-4" />+ </Button>+ }+ />+ );+}diff --git a/web/src/features/monitors/components/MonitorAutomationsPanel.toggle.clienttest.tsx b/web/src/features/monitors/components/MonitorAutomationsPanel.toggle.clienttest.tsxindex e77816113f..8d02164784 100644--- a/web/src/features/monitors/components/MonitorAutomationsPanel.toggle.clienttest.tsx+++ b/web/src/features/monitors/components/MonitorAutomationsPanel.toggle.clienttest.tsx@@ -24,7 +24,7 @@ vi.mock("@/src/utils/api", () => ({import { MonitorAutomationsPanel } from "./MonitorAutomationsPanel";-describe("MonitorAutomationsPanel toggle immutability", () => {+describe("MonitorAutomationsPanel selection", () => {it("does not mutate the memoized set: toggling the same unselected row twice reports it selected both times", () => {useQueryMock.mockReturnValue({ data: [automationRow], isPending: false });const onTriggerIdsChange = vi.fn();diff --git a/web/src/features/monitors/components/MonitorForm.clienttest.ts b/web/src/features/monitors/components/MonitorForm.clienttest.tsindex 6b388e80a6..ebdf13ad2f 100644--- a/web/src/features/monitors/components/MonitorForm.clienttest.ts+++ b/web/src/features/monitors/components/MonitorForm.clienttest.ts@@ -17,6 +17,7 @@ const {monitorToDefaults,nameOrPlaceholder,resolveViewChangePatch,+ monitorCreateAnalyticsProperties,} = __test;describe("createDefaults", () => {@@ -72,6 +73,28 @@ describe("createDefaults", () => {const defaults = createDefaults("project-1");expect(defaults.triggerIds).toEqual([]);});++ it("uses a prefilled evaluator score window", () => {+ expect(createDefaults("project-1", { window: "1d" }).window).toBe("1d");+ });+});++describe("monitorCreateAnalyticsProperties", () => {+ it("tracks the selected score window", () => {+ expect(+ monitorCreateAnalyticsProperties("evaluator_score", {+ view: "scores-numeric",+ metric: { measure: "value", aggregation: "avg" },+ window: "1d",+ }),+ ).toEqual({+ source: "evaluator_score",+ view: "scores-numeric",+ measure: "value",+ aggregation: "avg",+ window: "1d",+ });+ });});describe("nameOrPlaceholder", () => {diff --git a/web/src/features/monitors/components/MonitorForm.tsx b/web/src/features/monitors/components/MonitorForm.tsxindex 46453e3cae..06a69801cf 100644--- a/web/src/features/monitors/components/MonitorForm.tsx+++ b/web/src/features/monitors/components/MonitorForm.tsx@@ -7,6 +7,7 @@ import { zodResolver } from "@hookform/resolvers/zod";import { startCase } from "lodash";import { api } from "@/src/utils/api";+import { AIAssistedInput } from "@/src/components/ui/ai-assisted-input";import { Button } from "@/src/components/ui/button";import {Accordion,@@ -41,11 +42,15 @@ import {SelectValue,} from "@/src/components/ui/select";import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";+import { useLangfuseCloudRegion } from "@/src/features/organizations";+import { useProject } from "@/src/features/projects";import { showSuccessToast } from "@/src/features/notifications/showSuccessToast";import { showErrorToast } from "@/src/features/notifications/showErrorToast";import { WidgetPropertySelectItem } from "@/src/features/widgets/components/WidgetPropertySelectItem";import { MetricsFilterBuilder } from "@/src/features/metrics/components/MetricsFilterBuilder";import { partitionWidgetUiTableFiltersToView } from "@/src/features/dashboard/lib/dashboardUiTableToViewMapping";+import { usePostHogClientCapture } from "@/src/features/posthog-analytics";+import { resolveMonitorNameForSave } from "@/src/features/monitors/fns/resolveMonitorNameForSave";import { cn } from "@/src/utils/tailwind";import {@@ -88,18 +93,24 @@ import {} from "../helpers/renderMonitorLabels";/** createDefaults returns the form defaults for a brand-new monitor. */-const createDefaults = (projectId: string): Partial<CreateMonitor> => ({+const createDefaults = (+ projectId: string,+ prefill?: Partial<+ Pick<CreateMonitor, "view" | "filters" | "metric" | "window" | "tags">+ >,+ initialTriggerIds: string[] = [],+): Partial<CreateMonitor> => ({projectId,- view: "observations",- filters: [],- metric: { measure: "count", aggregation: "count" },- window: "5m",+ view: prefill?.view ?? "observations",+ filters: prefill?.filters ?? [],+ metric: prefill?.metric ?? { measure: "count", aggregation: "count" },+ window: prefill?.window ?? "5m",thresholdOperator: MonitorThresholdOperatorSchema.enum.GT,warningThreshold: null,noData: { mode: MonitorNoDataModeSchema.enum.SUBSTITUTE_ZERO },renotify: { mode: "OFF" },- tags: [],- triggerIds: [],+ tags: prefill?.tags ?? [],+ triggerIds: initialTriggerIds,status: MonitorStatusSchema.enum.ACTIVE,});@@ -140,19 +151,49 @@ const nameOrPlaceholder = (placeholder: string,): string => name || placeholder;+type MonitorAnalyticsSource =+ | "alerts"+ | "evaluator_score"+ | "evaluator_cost"+ | "all_evaluator_cost";++const monitorCreateAnalyticsProperties = (+ source: MonitorAnalyticsSource,+ monitor: Pick<CreateMonitor, "view" | "metric" | "window">,+) => ({+ source,+ view: monitor.view,+ measure: monitor.metric.measure,+ aggregation: monitor.metric.aggregation,+ window: monitor.window,+});+/** MonitorForm renders the create/edit form for a Monitor. */export const MonitorForm = ({projectId,monitor,+ prefill,+ analyticsSource = "alerts",+ initialTriggerIds = [],onNameChange,}: {projectId: string;monitor?: Monitor;+ prefill?: Partial<+ Pick<CreateMonitor, "view" | "filters" | "metric" | "window" | "tags">+ >;+ analyticsSource?: MonitorAnalyticsSource;+ initialTriggerIds?: string[];/** onNameChange fires on every form change so the host (e.g. the edit page header) can mirror the live name. */onNameChange?: (name: string) => void;}) => {/** router is the Next router used to redirect after a successful create. */const router = useRouter();+ const capture = usePostHogClientCapture();+ const { isLangfuseCloud } = useLangfuseCloudRegion();+ const { organization } = useProject(projectId);+ const nameAIAssistanceAvailable =+ isLangfuseCloud && Boolean(organization?.aiFeaturesEnabled);/** isEdit is true when the form is bound to an existing monitor. */const isEdit = Boolean(monitor);/** hasAccess gates write controls behind the alerts:CUD RBAC scope. */@@ -165,7 +206,7 @@ export const MonitorForm = ({/** defaultValues seeds the form from the existing monitor on edit, otherwise from createDefaults. */const defaultValues = isEdit? monitorToDefaults(monitor as Monitor)- : createDefaults(projectId);+ : createDefaults(projectId, prefill, initialTriggerIds);/** namePlaceholderRef holds the latest computed name placeholder for the resolver. */const namePlaceholderRef = useRef("");@@ -190,10 +231,42 @@ export const MonitorForm = ({mode: "onChange",});+ const suggestName = api.monitors.suggestName.useMutation();+ const generateNameSuggestion = async (): Promise<string | null> => {+ if (!nameAIAssistanceAvailable) return null;+ try {+ return await suggestName.mutateAsync({+ projectId,+ description: namePlaceholderRef.current,+ });+ } catch {+ return null;+ }+ };+ const requestNameSuggestion = async () => {+ const generatedName = await generateNameSuggestion();+ if (!generatedName) {+ showErrorToast(+ "Couldn't generate an alert title",+ "Please enter a title manually.",+ );+ return;+ }+ form.setValue("name", generatedName, {+ shouldDirty: true,+ shouldValidate: true,+ });+ onNameChange?.(generatedName);+ };+/** createMutation creates a new monitor and returns to the monitors list on success. */const createMutation = api.monitors.create.useMutation({onSuccess: async (_data, variables) => {await utils.monitors.invalidate();+ capture(+ "monitors:create",+ monitorCreateAnalyticsProperties(analyticsSource, variables),+ );showSuccessToast({title: "Alert created",description: `"${variables.name}" is now active.`,@@ -218,9 +291,24 @@ export const MonitorForm = ({/** onSubmit strips unsupported filter rows and dispatches the create or update mutation. */const onSubmit = form.handleSubmit(- (values) => {+ async (values) => {+ const resolvedName = await resolveMonitorNameForSave({+ name: form.getValues("name"),+ fallbackName: namePlaceholderRef.current,+ aiAvailable: nameAIAssistanceAvailable,+ generateName: generateNameSuggestion,+ });+ if (!resolvedName) {+ showErrorToast(+ "Couldn't generate an alert title",+ "Please enter a title manually and try again.",+ );+ return;+ }+const normalizedValues = {...values,+ name: resolvedName,filters: partitionWidgetUiTableFiltersToView(values.view as Parameters<typeof partitionWidgetUiTableFiltersToView@@ -327,7 +415,8 @@ export const MonitorForm = ({const submitting =form.formState.isSubmitting ||createMutation.isPending ||- updateMutation.isPending;+ updateMutation.isPending ||+ suggestName.isPending;return (<Form {...form}>@@ -709,18 +798,33 @@ export const MonitorForm = ({name="name"render={({ field }) => (<FormItem>- <FormLabel>Name</FormLabel>+ <FormLabel>Title</FormLabel><FormControl>- <Input+ <AIAssistedInput+ id="monitor-title"maxLength={200}placeholder={namePlaceholder}disabled={!hasAccess}- {...field}value={field.value ?? ""}onChange={(e) => {field.onChange(e);onNameChange?.(e.target.value ?? "");}}+ fieldName="title"+ aiAssistance={+ !nameAIAssistanceAvailable+ ? { state: "unavailable" }+ : suggestName.isPending+ ? { state: "generating" }+ : {+ state: "idle",+ onGenerate: () => {+ requestNameSuggestion().catch(+ () => undefined,+ );+ },+ }+ }/></FormControl><FormMessage />@@ -1017,4 +1121,5 @@ export const __test = {monitorToDefaults,nameOrPlaceholder,resolveViewChangePatch,+ monitorCreateAnalyticsProperties,};diff --git a/web/src/features/monitors/components/MonitorSeverityBadge.tsx b/web/src/features/monitors/components/MonitorSeverityBadge.tsxindex a1b93ae6a8..b28fbc32a7 100644--- a/web/src/features/monitors/components/MonitorSeverityBadge.tsx+++ b/web/src/features/monitors/components/MonitorSeverityBadge.tsx@@ -1,6 +1,4 @@/* eslint-disable @repo/no-style-props */-import { PauseCircle } from "lucide-react";-import { Badge, type BadgeProps } from "@/src/components/ui/badge";import { cn } from "@/src/utils/tailwind";import { type MonitorSeverity } from "@langfuse/shared/monitors";@@ -9,7 +7,7 @@ import { type MonitorSeverity } from "@langfuse/shared/monitors";const severityVariant: Record<MonitorSeverity, BadgeProps["variant"]> = {UNKNOWN: "outline",NO_DATA: "secondary",- PAUSED: "outline",+ PAUSED: "default",OK: "default",WARNING: "default",ALERT: "default",@@ -19,7 +17,7 @@ const severityVariant: Record<MonitorSeverity, BadgeProps["variant"]> = {const severityClassName: Record<MonitorSeverity, string> = {UNKNOWN: "text-gray-400",NO_DATA: "",- PAUSED: "text-gray-400",+ PAUSED: "bg-gray-500 text-white hover:bg-gray-500",OK: "bg-emerald-500 text-white hover:bg-emerald-500",WARNING: "bg-amber-600 text-white hover:bg-amber-600",ALERT: "bg-orange-600 text-white hover:bg-orange-600",@@ -52,12 +50,7 @@ export function MonitorSeverityBadge({className,)}>- {severity === "PAUSED" ? (- <span className="inline-flex items-center gap-1">- <PauseCircle className="h-4 w-4" />- {severityLabel.PAUSED}- </span>- ) : severity === "UNKNOWN" ? (+ {severity === "UNKNOWN" ? (<span className="inline-flex items-center gap-1.5"><spanclassName="h-3.5 w-3.5 animate-spin rounded-full border-2 border-gray-200 border-t-gray-400"diff --git a/web/src/features/monitors/components/MonitorsTable.clienttest.ts b/web/src/features/monitors/components/MonitorsTable.clienttest.tsindex d9ab283efd..631c02e56e 100644--- a/web/src/features/monitors/components/MonitorsTable.clienttest.ts+++ b/web/src/features/monitors/components/MonitorsTable.clienttest.ts@@ -9,6 +9,7 @@ import {MonitorThresholdOperatorSchema,} from "@langfuse/shared/monitors";+import { getMonitorFilterConfig } from "@/src/features/filters/config/monitors-config";import { __test } from "./MonitorsTable";const { filterStateToListMonitorFilter, buildStatusToggleUpdate } = __test;@@ -43,6 +44,21 @@ const monitorFixture = (status: Monitor["status"]): Monitor => ({lastCompletedAt: null,});+describe("getMonitorFilterConfig", () => {+ it("shows the evaluator facet only when evaluator-specific alerts exist", () => {+ expect(+ getMonitorFilterConfig(false).facets.some(+ (facet) => facet.column === "evaluatorId",+ ),+ ).toBe(false);+ expect(+ getMonitorFilterConfig(true).facets.some(+ (facet) => facet.column === "evaluatorId",+ ),+ ).toBe(true);+ });+});+describe("filterStateToListMonitorFilter", () => {it("passes severity `any of` through with values intact", () => {const state: FilterState = [@@ -140,6 +156,18 @@ describe("filterStateToListMonitorFilter", () => {]);});+ it("passes evaluator ID rows through unchanged", () => {+ const state: FilterState = [+ {+ type: "stringOptions",+ column: "evaluatorId",+ operator: "any of",+ value: ["eval-1"],+ },+ ];+ expect(filterStateToListMonitorFilter(state)).toEqual(state);+ });+it("passes tags rows through unchanged", () => {const state: FilterState = [{diff --git a/web/src/features/monitors/components/MonitorsTable.tsx b/web/src/features/monitors/components/MonitorsTable.tsxindex ba2a109894..f6e6fefea8 100644--- a/web/src/features/monitors/components/MonitorsTable.tsx+++ b/web/src/features/monitors/components/MonitorsTable.tsx@@ -18,7 +18,7 @@ import {DropdownMenuItem,DropdownMenuTrigger,} from "@/src/components/ui/dropdown-menu";-import { monitorFilterConfig } from "@/src/features/filters/config/monitors-config";+import { getMonitorFilterConfig } from "@/src/features/filters/config/monitors-config";import { useSidebarFilterState } from "@/src/features/filters/hooks/useSidebarFilterState";import { showErrorToast } from "@/src/features/notifications/showErrorToast";import { showSuccessToast } from "@/src/features/notifications/showSuccessToast";@@ -119,10 +119,17 @@ export function MonitorsTable() {displayValue: value.replace(/_/g, " "),})),tags: filterOptions.data?.tags.map((t) => ({ value: t.value })) ?? [],+ evaluatorId: filterOptions.data?.evaluators ?? [],}),[filterOptions.data],);+ const monitorFilterConfig = useMemo(+ () =>+ getMonitorFilterConfig((filterOptions.data?.evaluators.length ?? 0) > 0),+ [filterOptions.data?.evaluators.length],+ );+/** queryFilter is the bound sidebar filter state, synced to the URL and to session storage per project. */const queryFilter = useSidebarFilterState(monitorFilterConfig,diff --git a/web/src/features/monitors/fns/getInitialMonitorTriggerIds.clienttest.ts b/web/src/features/monitors/fns/getInitialMonitorTriggerIds.clienttest.tsnew file mode 100644index 0000000000..51a161519a--- /dev/null+++ b/web/src/features/monitors/fns/getInitialMonitorTriggerIds.clienttest.ts@@ -0,0 +1,20 @@+// @vitest-environment node++import { describe, expect, it } from "vitest";++import { getInitialMonitorTriggerIds } from "./getInitialMonitorTriggerIds";++describe("getInitialMonitorTriggerIds", () => {+ it("selects the first available automation", () => {+ expect(+ getInitialMonitorTriggerIds([+ { trigger: { id: "trigger-1" } },+ { trigger: { id: "trigger-2" } },+ ] as Parameters<typeof getInitialMonitorTriggerIds>[0]),+ ).toEqual(["trigger-1"]);+ });++ it("returns no selection without an available automation", () => {+ expect(getInitialMonitorTriggerIds([])).toEqual([]);+ });+});diff --git a/web/src/features/monitors/fns/getInitialMonitorTriggerIds.ts b/web/src/features/monitors/fns/getInitialMonitorTriggerIds.tsnew file mode 100644index 0000000000..c840bccc67--- /dev/null+++ b/web/src/features/monitors/fns/getInitialMonitorTriggerIds.ts@@ -0,0 +1,9 @@+import type { AutomationDomain } from "@langfuse/shared";++/** Selects the first available automation for a new monitor. */+export function getInitialMonitorTriggerIds(+ automations: AutomationDomain[],+): string[] {+ const firstTriggerId = automations[0]?.trigger.id;+ return firstTriggerId ? [firstTriggerId] : [];+}diff --git a/web/src/features/monitors/fns/getMonitorPrefill.clienttest.ts b/web/src/features/monitors/fns/getMonitorPrefill.clienttest.tsnew file mode 100644index 0000000000..11ce722529--- /dev/null+++ b/web/src/features/monitors/fns/getMonitorPrefill.clienttest.ts@@ -0,0 +1,96 @@+import { describe, expect, it } from "vitest";++import { getMonitorPrefill } from "@/src/features/monitors/fns/getMonitorPrefill";++const evaluatorFilters = [+ {+ column: "evaluatorId",+ type: "string",+ operator: "=",+ value: "evaluator-1",+ },+ {+ column: "isEvaluatorTest",+ type: "boolean",+ operator: "=",+ value: false,+ },+];++describe("getMonitorPrefill", () => {+ it.each([+ ["NUMERIC", "scores-numeric"],+ ["BOOLEAN", "scores-boolean"],+ ["CATEGORICAL", "scores-categorical"],+ ] as const)("prefills a %s evaluator score alert", (scoreDataType, view) => {+ expect(+ getMonitorPrefill({+ alert: "evaluatorScore",+ evaluatorId: "evaluator-1",+ scoreDataType,+ }),+ ).toMatchObject({+ view,+ filters: evaluatorFilters,+ window: "1d",+ tags: ["evaluators", "evaluator:evaluator-1"],+ metric:+ scoreDataType === "CATEGORICAL"+ ? { measure: "count", aggregation: "count" }+ : { measure: "value", aggregation: "avg" },+ });+ });++ it("prefills evaluator cost without evaluator test runs", () => {+ expect(+ getMonitorPrefill({+ alert: "evaluatorCost",+ evaluatorId: "evaluator-1",+ }),+ ).toEqual({+ view: "observations",+ filters: evaluatorFilters,+ metric: { measure: "totalCost", aggregation: "sum" },+ tags: ["evaluators", "evaluator:evaluator-1"],+ });+ });++ it("prefills aggregate evaluator cost across current and legacy executions", () => {+ expect(getMonitorPrefill({ alert: "allEvaluatorCost" })).toEqual({+ view: "observations",+ filters: [+ {+ column: "evaluatorId",+ type: "string",+ operator: "is not empty",+ value: "",+ },+ {+ column: "isEvaluatorTest",+ type: "boolean",+ operator: "=",+ value: false,+ },+ ],+ metric: { measure: "totalCost", aggregation: "sum" },+ tags: ["evaluators"],+ });+ });++ it.each([+ {},+ { alert: "unknown" },+ { alert: "evaluatorCost" },+ {+ alert: "evaluatorScore",+ evaluatorId: "evaluator-1",+ scoreDataType: "TEXT",+ },+ {+ alert: ["evaluatorCost"],+ evaluatorId: "evaluator-1",+ },+ ])("ignores invalid query values: %j", (query) => {+ expect(getMonitorPrefill(query)).toBeUndefined();+ });+});diff --git a/web/src/features/monitors/fns/getMonitorPrefill.ts b/web/src/features/monitors/fns/getMonitorPrefill.tsnew file mode 100644index 0000000000..3210f034d8--- /dev/null+++ b/web/src/features/monitors/fns/getMonitorPrefill.ts@@ -0,0 +1,93 @@+import type { CreateMonitor } from "@langfuse/shared/monitors";++const evaluatorIdFilter = (evaluatorId: string) => ({+ column: "evaluatorId",+ type: "string" as const,+ operator: "=" as const,+ value: evaluatorId,+});++const anyEvaluatorIdFilter = {+ column: "evaluatorId",+ type: "string" as const,+ operator: "is not empty" as const,+ value: "",+};++const excludeEvaluatorTestsFilter = {+ column: "isEvaluatorTest",+ type: "boolean" as const,+ operator: "=" as const,+ value: false,+};++const evaluatorAlertTags = ["evaluators"];+const specificEvaluatorAlertTags = (evaluatorId: string) => [+ ...evaluatorAlertTags,+ `evaluator:${evaluatorId}`,+];++/** Returns the supported alert-form defaults encoded by the new-alert URL. */+export function getMonitorPrefill(+ query: Record<string, string | string[] | undefined>,+): Partial<CreateMonitor> | undefined {+ const alert = query.alert;++ if (alert === "allEvaluatorCost") {+ return {+ view: "observations",+ filters: [anyEvaluatorIdFilter, excludeEvaluatorTestsFilter],+ metric: { measure: "totalCost", aggregation: "sum" },+ tags: evaluatorAlertTags,+ };+ }++ const evaluatorId = query.evaluatorId;+ if (typeof evaluatorId !== "string" || evaluatorId.length === 0) {+ return undefined;+ }++ if (alert === "evaluatorCost") {+ return {+ view: "observations",+ filters: [evaluatorIdFilter(evaluatorId), excludeEvaluatorTestsFilter],+ metric: { measure: "totalCost", aggregation: "sum" },+ tags: specificEvaluatorAlertTags(evaluatorId),+ };+ }++ if (alert !== "evaluatorScore") {+ return undefined;+ }++ const filters = [evaluatorIdFilter(evaluatorId), excludeEvaluatorTestsFilter];++ switch (query.scoreDataType) {+ case "NUMERIC":+ return {+ view: "scores-numeric",+ filters,+ metric: { measure: "value", aggregation: "avg" },+ window: "1d",+ tags: specificEvaluatorAlertTags(evaluatorId),+ };+ case "BOOLEAN":+ return {+ view: "scores-boolean",+ filters,+ metric: { measure: "value", aggregation: "avg" },+ window: "1d",+ tags: specificEvaluatorAlertTags(evaluatorId),+ };+ case "CATEGORICAL":+ return {+ view: "scores-categorical",+ filters,+ metric: { measure: "count", aggregation: "count" },+ window: "1d",+ tags: specificEvaluatorAlertTags(evaluatorId),+ };+ default:+ return undefined;+ }+}diff --git a/web/src/features/monitors/fns/invalidateMonitorQueriesAfterDelete.ts b/web/src/features/monitors/fns/invalidateMonitorQueriesAfterDelete.tsnew file mode 100644index 0000000000..e88a4d026a--- /dev/null+++ b/web/src/features/monitors/fns/invalidateMonitorQueriesAfterDelete.ts@@ -0,0 +1,16 @@+type MonitorQueryInvalidations = {+ all: { invalidate: () => Promise<unknown> };+ getFilterOptions: { invalidate: () => Promise<unknown> };+ linkedEvaluatorAlerts: { invalidate: () => Promise<unknown> };+};++/** Refreshes monitor collections without refetching the deleted monitor detail. */+export async function invalidateMonitorQueriesAfterDelete(+ monitors: MonitorQueryInvalidations,+): Promise<void> {+ await Promise.all([+ monitors.all.invalidate(),+ monitors.getFilterOptions.invalidate(),+ monitors.linkedEvaluatorAlerts.invalidate(),+ ]);+}diff --git a/web/src/features/monitors/fns/resolveMonitorNameForSave.clienttest.ts b/web/src/features/monitors/fns/resolveMonitorNameForSave.clienttest.tsnew file mode 100644index 0000000000..ab185bbd5a--- /dev/null+++ b/web/src/features/monitors/fns/resolveMonitorNameForSave.clienttest.ts@@ -0,0 +1,59 @@+// @vitest-environment node++import { describe, expect, it, vi } from "vitest";++import { resolveMonitorNameForSave } from "./resolveMonitorNameForSave";++describe("resolveMonitorNameForSave", () => {+ it("keeps a manually entered name", async () => {+ const generateName = vi.fn();++ await expect(+ resolveMonitorNameForSave({+ name: "Cost spike",+ fallbackName: "Fallback",+ aiAvailable: true,+ generateName,+ }),+ ).resolves.toBe("Cost spike");+ expect(generateName).not.toHaveBeenCalled();+ });++ it("generates an empty name when AI assistance is available", async () => {+ const generateName = vi.fn().mockResolvedValue(" Evaluator cost spike ");++ await expect(+ resolveMonitorNameForSave({+ name: "",+ fallbackName: "Fallback",+ aiAvailable: true,+ generateName,+ }),+ ).resolves.toBe("Evaluator cost spike");+ });++ it("uses the deterministic fallback when AI assistance is unavailable", async () => {+ const generateName = vi.fn();++ await expect(+ resolveMonitorNameForSave({+ name: "",+ fallbackName: "Count of observations is above 5",+ aiAvailable: false,+ generateName,+ }),+ ).resolves.toBe("Count of observations is above 5");+ expect(generateName).not.toHaveBeenCalled();+ });++ it("returns null when generation fails", async () => {+ await expect(+ resolveMonitorNameForSave({+ name: "",+ fallbackName: "Fallback",+ aiAvailable: true,+ generateName: vi.fn().mockResolvedValue(null),+ }),+ ).resolves.toBeNull();+ });+});diff --git a/web/src/features/monitors/fns/resolveMonitorNameForSave.ts b/web/src/features/monitors/fns/resolveMonitorNameForSave.tsnew file mode 100644index 0000000000..588a45dfa3--- /dev/null+++ b/web/src/features/monitors/fns/resolveMonitorNameForSave.ts@@ -0,0 +1,19 @@+/** Resolves a monitor name at save time, generating blank names when AI assistance is available. */+export async function resolveMonitorNameForSave({+ name,+ fallbackName,+ aiAvailable,+ generateName,+}: {+ name: string | undefined;+ fallbackName: string;+ aiAvailable: boolean;+ generateName: () => Promise<string | null>;+}): Promise<string | null> {+ const enteredName = name?.trim();+ if (enteredName) return enteredName;+ if (!aiAvailable) return fallbackName;++ const generatedName = (await generateName())?.trim();+ return generatedName || null;+}diff --git a/web/src/features/monitors/index.ts b/web/src/features/monitors/index.tsnew file mode 100644index 0000000000..9b22c7fc62--- /dev/null+++ b/web/src/features/monitors/index.ts@@ -0,0 +1 @@+export { MonitorSeverityBadge } from "@/src/features/monitors/components/MonitorSeverityBadge";diff --git a/web/src/features/monitors/pages/EditMonitorPage.tsx b/web/src/features/monitors/pages/EditMonitorPage.tsxindex 1df94c7e00..c31af08b95 100644--- a/web/src/features/monitors/pages/EditMonitorPage.tsx+++ b/web/src/features/monitors/pages/EditMonitorPage.tsx@@ -3,8 +3,13 @@ import { useState } from "react";import { ErrorPage } from "@/src/components/error-page";import Page from "@/src/components/layouts/page";+import { DeleteMonitorButton } from "@/src/features/monitors/components/DeleteMonitorButton";import { MonitorForm } from "@/src/features/monitors/components/MonitorForm";import { MonitorPagePermissions } from "@/src/features/monitors/components/MonitorPagePermissions";+import { invalidateMonitorQueriesAfterDelete } from "@/src/features/monitors/fns/invalidateMonitorQueriesAfterDelete";+import { showErrorToast } from "@/src/features/notifications/showErrorToast";+import { showSuccessToast } from "@/src/features/notifications/showSuccessToast";+import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";import { api, type APIError } from "@/src/utils/api";import { type Monitor } from "@langfuse/shared/monitors";@@ -41,10 +46,44 @@ function EditMonitorPageRouter() {/** EditMonitorFormPage renders the edit monitors form */const EditMonitorFormPage = ({ monitor }: { monitor: Monitor }) => {+ const router = useRouter();+ const utils = api.useUtils();const [liveName, setLiveName] = useState(monitor.name);+ const canDelete = useHasProjectAccess({+ projectId: monitor.projectId,+ scope: "alerts:CUD",+ });+ const deleteMonitor = api.monitors.delete.useMutation({+ onSuccess: async () => {+ await invalidateMonitorQueriesAfterDelete(utils.monitors);+ showSuccessToast({+ title: "Alert deleted",+ description: `"${monitor.name}" has been deleted.`,+ });+ await router.replace(`/project/${monitor.projectId}/alerts`);+ },+ onError: (error) => showErrorToast("Failed to delete alert", error.message),+ });return (- <Page withPadding headerProps={getHeaderProps(monitor.projectId, liveName)}>+ <Page+ withPadding+ headerProps={{+ ...getHeaderProps(monitor.projectId, liveName),+ actionButtonsRight: canDelete ? (+ <DeleteMonitorButton+ monitorName={monitor.name}+ deleting={deleteMonitor.isPending}+ onDelete={async () => {+ await deleteMonitor.mutateAsync({+ projectId: monitor.projectId,+ id: monitor.id,+ });+ }}+ />+ ) : undefined,+ }}+ ><MonitorFormprojectId={monitor.projectId}monitor={monitor}diff --git a/web/src/features/monitors/pages/NewMonitorPage.tsx b/web/src/features/monitors/pages/NewMonitorPage.tsxindex fb255ede59..57a20c2d4d 100644--- a/web/src/features/monitors/pages/NewMonitorPage.tsx+++ b/web/src/features/monitors/pages/NewMonitorPage.tsx@@ -1,13 +1,42 @@import { useRouter } from "next/router";import Page from "@/src/components/layouts/page";+import { Skeleton } from "@/src/components/ui/skeleton";import { MonitorForm } from "@/src/features/monitors/components/MonitorForm";import { MonitorPagePermissions } from "@/src/features/monitors/components/MonitorPagePermissions";+import { getInitialMonitorTriggerIds } from "@/src/features/monitors/fns/getInitialMonitorTriggerIds";+import { getMonitorPrefill } from "@/src/features/monitors/fns/getMonitorPrefill";+import { api } from "@/src/utils/api";+import { TriggerEventSource } from "@langfuse/shared";++const alertAnalyticsSource = (alert: string | string[] | undefined) => {+ switch (alert) {+ case "evaluatorScore":+ return "evaluator_score" as const;+ case "evaluatorCost":+ return "evaluator_cost" as const;+ case "allEvaluatorCost":+ return "all_evaluator_cost" as const;+ default:+ return "alerts" as const;+ }+};/** NewMonitorPage renders the create-monitor form for a project. */export default function NewMonitorPage() {const router = useRouter();const projectId = router.query.projectId as string;+ const prefill = router.isReady ? getMonitorPrefill(router.query) : undefined;+ const analyticsSource = alertAnalyticsSource(router.query.alert);+ const automations = api.automations.getAutomations.useQuery(+ { projectId, eventSource: TriggerEventSource.Monitor },+ {+ enabled: Boolean(router.isReady && projectId),+ trpc: { context: { skipBatch: true } },+ refetchOnWindowFocus: false,+ },+ );+ const initialTriggerIds = getInitialMonitorTriggerIds(automations.data ?? []);return (<MonitorPagePermissions scope="alerts:CUD">@@ -20,7 +49,16 @@ export default function NewMonitorPage() {],}}>- <MonitorForm projectId={projectId} />+ {router.isReady && !automations.isPending ? (+ <MonitorForm+ projectId={projectId}+ prefill={prefill}+ analyticsSource={analyticsSource}+ initialTriggerIds={initialTriggerIds}+ />+ ) : (+ <Skeleton className="h-96 w-full" />+ )}</Page></MonitorPagePermissions>);diff --git a/web/src/features/monitors/server/index.ts b/web/src/features/monitors/server/index.tsnew file mode 100644index 0000000000..6dee3a99a5--- /dev/null+++ b/web/src/features/monitors/server/index.ts@@ -0,0 +1,2 @@+export { listLinkedEvaluatorAlerts } from "@/src/features/monitors/server/listLinkedEvaluatorAlerts";+export { suggestMonitorName } from "@/src/features/monitors/server/suggestMonitorName";diff --git a/web/src/features/monitors/server/listLinkedEvaluatorAlerts.servertest.ts b/web/src/features/monitors/server/listLinkedEvaluatorAlerts.servertest.tsnew file mode 100644index 0000000000..49091629f1--- /dev/null+++ b/web/src/features/monitors/server/listLinkedEvaluatorAlerts.servertest.ts@@ -0,0 +1,115 @@+import { describe, expect, it, vi } from "vitest";++import { listLinkedEvaluatorAlerts } from "./listLinkedEvaluatorAlerts";++const prismaWithFindMany = (rows: unknown[] = [], linkedIds: string[] = []) => {+ const findMany = vi.fn().mockResolvedValue(rows);+ const $queryRaw = vi.fn().mockResolvedValue(linkedIds.map((id) => ({ id })));+ return {+ findMany,+ $queryRaw,+ prisma: { monitor: { findMany }, $queryRaw } as unknown as Parameters<+ typeof listLinkedEvaluatorAlerts+ >[0],+ };+};++describe("listLinkedEvaluatorAlerts", () => {+ it("filters evaluator connections in the database", async () => {+ const { prisma, findMany, $queryRaw } = prismaWithFindMany(+ [],+ ["monitor-1"],+ );++ await listLinkedEvaluatorAlerts(prisma, {+ scope: "evaluator",+ projectId: "project-1",+ evaluatorId: "evaluator-1",+ });++ expect(findMany).toHaveBeenCalledWith(+ expect.objectContaining({+ where: {+ projectId: "project-1",+ id: { in: ["monitor-1"] },+ },+ }),+ );+ expect($queryRaw).toHaveBeenCalledOnce();+ });++ it("returns at most 20 alerts and reports when more exist", async () => {+ const rows = Array.from({ length: 21 }, (_, index) => ({+ id: `alert-${index}`,+ name: `Alert ${index}`,+ status: "ACTIVE",+ severity: "UNKNOWN",+ metric: { measure: "count", aggregation: "count" },+ thresholdOperator: "GT",+ alertThreshold: { toNumber: () => index },+ alertedAt: null,+ }));+ const { prisma, findMany } = prismaWithFindMany(+ rows,+ rows.map((row) => row.id),+ );++ const result = await listLinkedEvaluatorAlerts(prisma, {+ scope: "evaluator",+ projectId: "project-1",+ evaluatorId: "evaluator-1",+ });++ expect(findMany).toHaveBeenCalledWith(+ expect.objectContaining({ take: 21 }),+ );+ expect(result.data).toHaveLength(20);+ expect(result.hasMore).toBe(true);+ });++ it("filters aggregate evaluator cost alerts in the database", async () => {+ const { prisma, findMany } = prismaWithFindMany();++ await listLinkedEvaluatorAlerts(prisma, {+ scope: "allEvaluators",+ projectId: "project-1",+ });++ expect(findMany).toHaveBeenCalledWith(+ expect.objectContaining({+ where: {+ projectId: "project-1",+ view: "OBSERVATIONS",+ AND: [+ { metric: { path: ["measure"], equals: "totalCost" } },+ { metric: { path: ["aggregation"], equals: "sum" } },+ {+ filters: {+ array_contains: [+ {+ column: "evaluatorId",+ type: "string",+ operator: "is not empty",+ value: "",+ },+ ],+ },+ },+ {+ filters: {+ array_contains: [+ {+ column: "isEvaluatorTest",+ type: "boolean",+ operator: "=",+ value: false,+ },+ ],+ },+ },+ ],+ },+ }),+ );+ });+});diff --git a/web/src/features/monitors/server/listLinkedEvaluatorAlerts.ts b/web/src/features/monitors/server/listLinkedEvaluatorAlerts.tsnew file mode 100644index 0000000000..3ff4e9169d--- /dev/null+++ b/web/src/features/monitors/server/listLinkedEvaluatorAlerts.ts@@ -0,0 +1,106 @@+import { metric as MetricSchema } from "@langfuse/shared";+import { findMonitorIdsLinkedToEvaluators } from "@langfuse/shared/monitors/server";+import type { Prisma, PrismaClient } from "@langfuse/shared/src/db";++const LINKED_ALERTS_PAGE_SIZE = 20;++/** Lists the first page of monitors connected to an evaluator alert scope. */+export async function listLinkedEvaluatorAlerts(+ prisma: PrismaClient,+ params:+ | { scope: "evaluator"; projectId: string; evaluatorId: string }+ | { scope: "allEvaluators"; projectId: string },+) {+ const evaluatorMonitorIds =+ params.scope === "evaluator"+ ? await findMonitorIdsLinkedToEvaluators(prisma, {+ projectId: params.projectId,+ evaluatorIds: [params.evaluatorId],+ limit: LINKED_ALERTS_PAGE_SIZE + 1,+ })+ : undefined;+ const where: Prisma.MonitorWhereInput =+ params.scope === "evaluator"+ ? {+ projectId: params.projectId,+ id: { in: evaluatorMonitorIds },+ }+ : {+ projectId: params.projectId,+ view: "OBSERVATIONS",+ AND: [+ { metric: { path: ["measure"], equals: "totalCost" } },+ { metric: { path: ["aggregation"], equals: "sum" } },+ {+ filters: {+ array_contains: [+ {+ column: "evaluatorId",+ type: "string",+ operator: "is not empty",+ value: "",+ },+ ],+ },+ },+ {+ filters: {+ array_contains: [+ {+ column: "isEvaluatorTest",+ type: "boolean",+ operator: "=",+ value: false,+ },+ ],+ },+ },+ ],+ };++ const monitors =+ evaluatorMonitorIds?.length === 0+ ? []+ : await prisma.monitor.findMany({+ where,+ select: {+ id: true,+ name: true,+ status: true,+ severity: true,+ metric: true,+ thresholdOperator: true,+ alertThreshold: true,+ alertedAt: true,+ },+ orderBy: { updatedAt: "desc" },+ take: LINKED_ALERTS_PAGE_SIZE + 1,+ });++ return {+ data: monitors+ .slice(0, LINKED_ALERTS_PAGE_SIZE)+ .map(+ ({+ id,+ name,+ status,+ severity,+ metric,+ thresholdOperator,+ alertThreshold,+ alertedAt,+ }) => ({+ id,+ name,+ status,+ severity,+ metric: MetricSchema.parse(metric),+ thresholdOperator,+ alertThreshold: alertThreshold.toNumber(),+ alertedAt,+ }),+ ),+ hasMore: monitors.length > LINKED_ALERTS_PAGE_SIZE,+ };+}diff --git a/web/src/features/monitors/server/suggestMonitorName.ts b/web/src/features/monitors/server/suggestMonitorName.tsnew file mode 100644index 0000000000..488e62a7c0--- /dev/null+++ b/web/src/features/monitors/server/suggestMonitorName.ts@@ -0,0 +1,57 @@+import type { PrismaClient } from "@langfuse/shared/src/db";+import {+ ChatMessageRole,+ ChatMessageType,+ generateLangfuseAIText,+ getClientInitiatedNonStreamingLlmTimeoutMs,+ logger,+} from "@langfuse/shared/src/server";++import { resolveLangfuseAiFeatureAvailability } from "@/src/features/ai-features/server";++/** Generates a concise monitor name when Langfuse AI features are available. */+export async function suggestMonitorName({+ prisma,+ projectId,+ description,+}: {+ prisma: PrismaClient;+ projectId: string;+ description: string;+}): Promise<string | null> {+ const availability = await resolveLangfuseAiFeatureAvailability({+ prisma,+ projectId,+ });+ if (!availability.available) return null;++ try {+ const generated = await generateLangfuseAIText({+ messages: [+ {+ role: ChatMessageRole.System,+ content:+ "Return only a concise alert title of at most six words. Describe what is monitored and the trigger. Do not use quotes or punctuation at the end.",+ type: ChatMessageType.System,+ },+ {+ role: ChatMessageRole.User,+ content: description,+ type: ChatMessageType.User,+ },+ ],+ model: availability.model,+ maxTokens: 40,+ timeout: getClientInitiatedNonStreamingLlmTimeoutMs(),+ });+ return (+ generated+ ?.trim()+ .replace(/^['"]|['"]$/g, "")+ .slice(0, 200) || null+ );+ } catch (error) {+ logger.warn("Alert title generation failed", { projectId, error });+ return null;+ }+}diff --git a/web/src/features/organizations/index.ts b/web/src/features/organizations/index.tsnew file mode 100644index 0000000000..23e38ca77e--- /dev/null+++ b/web/src/features/organizations/index.ts@@ -0,0 +1 @@+export { useLangfuseCloudRegion } from "@/src/features/organizations/hooks";diff --git a/web/src/features/posthog-analytics/usePostHogClientCapture.ts b/web/src/features/posthog-analytics/usePostHogClientCapture.tsindex ca1e7f6b81..16e9a424f0 100644--- a/web/src/features/posthog-analytics/usePostHogClientCapture.ts+++ b/web/src/features/posthog-analytics/usePostHogClientCapture.ts@@ -185,6 +185,7 @@ const events = {"empty_state_template_select","empty_state_browse_library","empty_state_detect_topics",+ "alert_create_clicked",],evaluation_rules: ["create",@@ -245,7 +246,7 @@ const events = {"delete_dashboard_form_open","delete_dashboard_button_click",],- monitors: ["delete_form_open", "delete_monitor_button_click"],+ monitors: ["create", "delete_form_open", "delete_monitor_button_click"],datasets: ["delete_form_open","delete_dataset_button_click",diff --git a/web/src/features/projects/index.ts b/web/src/features/projects/index.tsnew file mode 100644index 0000000000..1e0fac144a--- /dev/null+++ b/web/src/features/projects/index.ts@@ -0,0 +1 @@+export { useProject } from "@/src/features/projects/hooks";diff --git a/web/src/features/widgets/components/WidgetForm.observationNameOptions.clienttest.tsx b/web/src/features/widgets/components/WidgetForm.observationNameOptions.clienttest.tsxindex 5ce5f21bbe..9d74829fa1 100644--- a/web/src/features/widgets/components/WidgetForm.observationNameOptions.clienttest.tsx+++ b/web/src/features/widgets/components/WidgetForm.observationNameOptions.clienttest.tsx@@ -26,6 +26,7 @@ vi.mock("@/src/utils/api", () => ({filterOptions: { useQuery: generationsFilterOptionsUseQuery },},events: { filterOptions: { useQuery: noopQuery } },+ evalsV2: { options: { useQuery: noopQuery } },projects: { environmentFilterOptions: { useQuery: noopQuery } },datasets: { allDatasetMeta: { useQuery: noopQuery } },dashboard: { executeQuery: { useQuery: noopQuery } },diff --git a/web/src/server/api/routers/monitors.ts b/web/src/server/api/routers/monitors.tsindex 4b9002daed..01ae4cb7a1 100644--- a/web/src/server/api/routers/monitors.ts+++ b/web/src/server/api/routers/monitors.ts@@ -7,6 +7,10 @@ import {} from "@/src/server/api/trpc";import { throwIfNoProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";import { createWithinEntitlementLimit } from "@/src/features/entitlements/server/createWithinEntitlementLimit";+import {+ listLinkedEvaluatorAlerts,+ suggestMonitorName,+} from "@/src/features/monitors/server";import {CreateMonitorSchema,DeleteMonitorSchema,@@ -109,6 +113,55 @@ export const monitorsRouter = createTRPCRouter({return { count };}),+ suggestName: monitorsProcedure+ .input(+ z.object({+ projectId: z.string(),+ description: z.string().min(1).max(500),+ }),+ )+ .mutation(async ({ ctx, input }) => {+ throwIfNoProjectAccess({+ session: ctx.session,+ projectId: input.projectId,+ scope: "alerts:CUD",+ });+ return suggestMonitorName({+ prisma: ctx.prisma,+ projectId: input.projectId,+ description: input.description,+ });+ }),++ linkedEvaluatorAlerts: monitorsProcedure+ .input(z.object({ projectId: z.string(), evaluatorId: z.string() }))+ .query(async ({ ctx, input }) => {+ throwIfNoProjectAccess({+ session: ctx.session,+ projectId: input.projectId,+ scope: "alerts:read",+ });+ return listLinkedEvaluatorAlerts(ctx.prisma, {+ scope: "evaluator",+ projectId: input.projectId,+ evaluatorId: input.evaluatorId,+ });+ }),++ linkedAllEvaluatorSpendAlerts: monitorsProcedure+ .input(z.object({ projectId: z.string() }))+ .query(async ({ ctx, input }) => {+ throwIfNoProjectAccess({+ session: ctx.session,+ projectId: input.projectId,+ scope: "alerts:read",+ });+ return listLinkedEvaluatorAlerts(ctx.prisma, {+ scope: "allEvaluators",+ projectId: input.projectId,+ });+ }),+/** hasAny reports whether the project owns at least one monitor; drives the list-page onboarding splash. */hasAny: monitorsProcedure.input(z.object({ projectId: z.string() }))