🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(evals): link code evaluators to their scores (#16438)
Tobias Wochingerc23526cbee5c
diff --git a/packages/shared/src/server/tableMappings/mapScoresTable.ts b/packages/shared/src/server/tableMappings/mapScoresTable.tsindex 804c4cc921..cd2ddbb7a6 100644--- a/packages/shared/src/server/tableMappings/mapScoresTable.ts+++ b/packages/shared/src/server/tableMappings/mapScoresTable.ts@@ -59,6 +59,13 @@ export const scoresTableUiColumnDefinitions: UiColumnMappings = [clickhouseSelect: "name",queryPrefix: "s",},+ {+ uiTableName: "Evaluator ID",+ uiTableId: "evaluatorId",+ clickhouseTableName: "scores",+ clickhouseSelect:+ "coalesce(nullIf(s.metadata['evaluator_id'], ''), s.metadata['job_configuration_id'])",+ },{uiTableName: "Value",uiTableId: "value",diff --git a/packages/shared/src/tableDefinitions/scoresTable.ts b/packages/shared/src/tableDefinitions/scoresTable.tsindex 02d7341d41..6a09daaac5 100644--- a/packages/shared/src/tableDefinitions/scoresTable.ts+++ b/packages/shared/src/tableDefinitions/scoresTable.ts@@ -62,6 +62,14 @@ export const scoresTableCols: ColumnDefinition[] = [internal: 's."name"',options: [],},+ {+ name: "Evaluator ID",+ id: "evaluatorId",+ type: "stringOptions",+ // ClickHouse-only virtual column; see mapScoresTable.ts.+ internal: "",+ options: [],+ },{ name: "Value", id: "value", type: "number", internal: 's."value"' },{name: "Boolean Value",diff --git a/web/src/__tests__/server/scores-trpc.servertest.ts b/web/src/__tests__/server/scores-trpc.servertest.tsindex b09d4ec991..c6238d4b77 100644--- a/web/src/__tests__/server/scores-trpc.servertest.ts+++ b/web/src/__tests__/server/scores-trpc.servertest.ts@@ -195,6 +195,76 @@ describe("scores trpc", () => {);});+ it("filters evaluator scores by recorded evaluator and legacy rule metadata", async () => {+ const [evaluator, otherEvaluator] = await Promise.all(+ ["Evaluator", "Other evaluator"].map((name) =>+ prisma.evaluator.create({+ data: { projectId, name, type: "CODE" },+ }),+ ),+ );+ const rule = await prisma.evaluationRule.create({+ data: {+ projectId,+ name: "Legacy rule",+ targetObject: "EVENT",+ filter: [],+ sampling: 1,+ delay: 0,+ assignments: {+ create: { projectId, evaluatorId: evaluator.id },+ },+ },+ });+ const directScore = createTraceScore({+ project_id: projectId,+ name: "direct-score",+ metadata: { evaluator_id: evaluator.id },+ });+ const legacyScore = createTraceScore({+ project_id: projectId,+ name: "legacy-score-with-a-different-name",+ metadata: { job_configuration_id: rule.id },+ });+ const otherScore = createTraceScore({+ project_id: projectId,+ name: "other-score",+ metadata: { evaluator_id: otherEvaluator.id },+ });+ await createScoresCh([directScore, legacyScore, otherScore]);++ const payload = {+ projectId,+ filter: [+ {+ column: "evaluatorId",+ type: "stringOptions" as const,+ operator: "any of" as const,+ value: [evaluator.id, rule.id],+ },+ ],+ orderBy: { column: "timestamp", order: "DESC" as const },+ page: 0,+ limit: 50,+ };++ const [scores, eventScores, count, eventCount] = await Promise.all([+ caller.scores.all(payload),+ caller.scores.allFromEvents(payload),+ caller.scores.countAll({ ...payload, orderBy: null }),+ caller.scores.countAllFromEvents({ ...payload, orderBy: null }),+ ]);++ expect(new Set(scores.scores.map(({ id }) => id))).toEqual(+ new Set([directScore.id, legacyScore.id]),+ );+ expect(new Set(eventScores.scores.map(({ id }) => id))).toEqual(+ new Set([directScore.id, legacyScore.id]),+ );+ expect(count.totalCount).toBe(2);+ expect(eventCount.totalCount).toBe(2);+ });+it("does not match empty boolean representations when filtering boolean values", async () => {const trueBooleanScore = createTraceScore({project_id: projectId,diff --git a/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.clienttest.ts b/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.clienttest.tsindex 0f953fdc48..e54ececc06 100644--- a/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.clienttest.ts+++ b/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.clienttest.ts@@ -11,20 +11,54 @@ import {} from "./evaluatorScoresUrl";describe("evaluatorScoresUrl", () => {- it("shows evaluator scores across all environments", () => {+ it("filters code evaluator scores by evaluator and assigned rule IDs", () => {const url = new URL(- evaluatorScoresUrl("project-1", "evaluator-name"),+ evaluatorScoresUrl(+ "project-1",+ "evaluator-id",+ "Code evaluator",+ EvalTemplateTypeEnum.CODE,+ ["rule-1", "rule-2"],+ ),"https://langfuse.local",);expect(url.pathname).toBe("/project/project-1/scores");expect(url.searchParams.get("showAllEnvironments")).toBe("true");+ expect(decodeFiltersGeneric(url.searchParams.get("filter") ?? "")).toEqual([+ {+ column: "evaluatorId",+ type: "stringOptions",+ operator: "any of",+ value: ["evaluator-id", "rule-1", "rule-2"],+ },+ {+ column: "source",+ type: "stringOptions",+ operator: "any of",+ value: ["EVAL"],+ },+ ]);+ });++ it("filters judge scores by evaluator name", () => {+ const url = new URL(+ evaluatorScoresUrl(+ "project-1",+ "evaluator-id",+ "Correctness",+ EvalTemplateTypeEnum.LLM_AS_JUDGE,+ ["rule-1"],+ ),+ "https://langfuse.local",+ );+expect(decodeFiltersGeneric(url.searchParams.get("filter") ?? "")).toEqual([{column: "name",type: "stringOptions",operator: "any of",- value: ["evaluator-name"],+ value: ["Correctness"],},{column: "source",diff --git a/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.ts b/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.tsindex 683d79cca7..8116310b46 100644--- a/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.ts+++ b/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.ts@@ -6,14 +6,30 @@ import {encodeFiltersGeneric,} from "@langfuse/shared";-export function evaluatorScoresUrl(projectId: string, name: string) {+export function evaluatorScoresUrl(+ projectId: string,+ evaluatorId: string,+ evaluatorName: string,+ evaluatorType: EvalTemplateType,+ assignedRuleIds: string[],+) {+ const evaluatorFilter: FilterState[number] =+ evaluatorType === EvalTemplateTypeEnum.CODE+ ? {+ column: "evaluatorId",+ type: "stringOptions",+ operator: "any of",+ value: [...new Set([evaluatorId, ...assignedRuleIds])],+ }+ : {+ column: "name",+ type: "stringOptions",+ operator: "any of",+ value: [evaluatorName],+ };+const filter: FilterState = [- {- column: "name",- type: "stringOptions",- operator: "any of",- value: [name],- },+ evaluatorFilter,{column: "source",type: "stringOptions",diff --git a/web/src/features/evals/v2/pages/EvaluatorsPage.tsx b/web/src/features/evals/v2/pages/EvaluatorsPage.tsxindex adfdb11aeb..11ea4d8ee0 100644--- a/web/src/features/evals/v2/pages/EvaluatorsPage.tsx+++ b/web/src/features/evals/v2/pages/EvaluatorsPage.tsx@@ -443,7 +443,15 @@ export default function EvaluatorsPage() {<EvaluatorActionsCellcanViewExecutions={hasExecutionReadAccess}onViewScores={() =>- router.push(evaluatorScoresUrl(projectId, row.original.name))+ router.push(+ evaluatorScoresUrl(+ projectId,+ row.original.id,+ row.original.name,+ row.original.type,+ row.original.assignedRuleIds,+ ),+ )}onViewExecutions={() =>router.push(diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts b/web/src/features/evals/v2/server/evaluators/evaluatorRepository.tsindex 7391fa9b58..35c3706f6a 100644--- a/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts+++ b/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts@@ -180,7 +180,7 @@ export async function listEvaluators(params: {assignments: {where: { projectId: params.projectId },select: {- evaluationRule: { select: { status: true } },+ evaluationRule: { select: { id: true, status: true } },},},},@@ -190,6 +190,9 @@ export async function listEvaluators(params: {return {evaluators: evaluators.map(({ assignments, ...evaluator }) => ({...evaluator,+ assignedRuleIds: assignments.map(+ ({ evaluationRule }) => evaluationRule.id,+ ),hasActiveRules: assignments.some(({ evaluationRule }) => evaluationRule.status === "ACTIVE",),