🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(evals): link decision-model evaluators to their scores by evaluator ID (#17954)
Hassieb Pakzadfd5c9ee18e07
diff --git a/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.clienttest.ts b/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.clienttest.tsindex e11accf48c..751abf0ea3 100644--- a/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.clienttest.ts+++ b/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.clienttest.ts@@ -11,34 +11,39 @@ import {} from "./evaluatorScoresUrl";describe("evaluatorScoresUrl", () => {- it("filters code evaluator scores by evaluator ID", () => {- const url = new URL(- evaluatorScoresUrl(- "project-1",- "evaluator-id",- "Code evaluator",- EvalTemplateTypeEnum.CODE,- ),- "https://langfuse.local",- );+ it.each([EvalTemplateTypeEnum.CODE, EvalTemplateTypeEnum.DECISION_MODEL])(+ "filters %s evaluator scores by evaluator ID",+ (evaluatorType) => {+ const url = new URL(+ evaluatorScoresUrl(+ "project-1",+ "evaluator-id",+ "Evaluator",+ evaluatorType,+ ),+ "https://langfuse.local",+ );- expect(url.pathname).toBe("/project/project-1/scores");- expect(url.searchParams.get("showAllEnvironments")).toBe("true");- expect(decodeFiltersGeneric(url.searchParams.get("filter") ?? "")).toEqual([- {- column: "evaluatorId",- type: "stringOptions",- operator: "any of",- value: ["evaluator-id"],- },- {- column: "source",- type: "stringOptions",- operator: "any of",- value: ["EVAL"],- },- ]);- });+ expect(url.pathname).toBe("/project/project-1/scores");+ expect(url.searchParams.get("showAllEnvironments")).toBe("true");+ expect(+ decodeFiltersGeneric(url.searchParams.get("filter") ?? ""),+ ).toEqual([+ {+ column: "evaluatorId",+ type: "stringOptions",+ operator: "any of",+ value: ["evaluator-id"],+ },+ {+ column: "source",+ type: "stringOptions",+ operator: "any of",+ value: ["EVAL"],+ },+ ]);+ },+ );it("filters judge scores by evaluator name", () => {const url = new URL(diff --git a/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.ts b/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.tsindex 486b2f2745..38c3a83613 100644--- a/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.ts+++ b/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.ts@@ -12,19 +12,23 @@ export function evaluatorScoresUrl(evaluatorName: string,evaluatorType: EvalTemplateType,) {+ // Code and decision-model evaluators name their scores themselves (per+ // question for decision models), so only the evaluator ID finds them.+ // Judge scores are named after the evaluator, and name also matches judge+ // scores written before scores carried an evaluator ID.const evaluatorFilter: FilterState[number] =- evaluatorType === EvalTemplateTypeEnum.CODE+ evaluatorType === EvalTemplateTypeEnum.LLM_AS_JUDGE? {- column: "evaluatorId",+ column: "name",type: "stringOptions",operator: "any of",- value: [evaluatorId],+ value: [evaluatorName],}: {- column: "name",+ column: "evaluatorId",type: "stringOptions",operator: "any of",- value: [evaluatorName],+ value: [evaluatorId],};const filter: FilterState = [