Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(evals): link decision-model evaluators to their scores by evaluator ID (#17954)

Hassieb Pakzadfd5c9ee18e07
diff --git a/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.clienttest.ts b/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.clienttest.ts
index e11accf48c..751abf0ea3 100644
--- a/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.clienttest.ts
+++ b/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.clienttest.ts
@@ -11,34 +11,39 @@ import {
} from "./evaluatorScoresUrl";
describe("evaluatorScoresUrl", () => {
- it("filters code evaluator scores by evaluator ID", () => {
- const url = new URL(
- evaluatorScoresUrl(
- "project-1",
- "evaluator-id",
- "Code evaluator",
- EvalTemplateTypeEnum.CODE,
- ),
- "https://langfuse.local",
- );
+ it.each([EvalTemplateTypeEnum.CODE, EvalTemplateTypeEnum.DECISION_MODEL])(
+ "filters %s evaluator scores by evaluator ID",
+ (evaluatorType) => {
+ const url = new URL(
+ evaluatorScoresUrl(
+ "project-1",
+ "evaluator-id",
+ "Evaluator",
+ evaluatorType,
+ ),
+ "https://langfuse.local",
+ );
- expect(url.pathname).toBe("/project/project-1/scores");
- expect(url.searchParams.get("showAllEnvironments")).toBe("true");
- expect(decodeFiltersGeneric(url.searchParams.get("filter") ?? "")).toEqual([
- {
- column: "evaluatorId",
- type: "stringOptions",
- operator: "any of",
- value: ["evaluator-id"],
- },
- {
- column: "source",
- type: "stringOptions",
- operator: "any of",
- value: ["EVAL"],
- },
- ]);
- });
+ expect(url.pathname).toBe("/project/project-1/scores");
+ expect(url.searchParams.get("showAllEnvironments")).toBe("true");
+ expect(
+ decodeFiltersGeneric(url.searchParams.get("filter") ?? ""),
+ ).toEqual([
+ {
+ column: "evaluatorId",
+ type: "stringOptions",
+ operator: "any of",
+ value: ["evaluator-id"],
+ },
+ {
+ column: "source",
+ type: "stringOptions",
+ operator: "any of",
+ value: ["EVAL"],
+ },
+ ]);
+ },
+ );
it("filters judge scores by evaluator name", () => {
const url = new URL(
diff --git a/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.ts b/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.ts
index 486b2f2745..38c3a83613 100644
--- a/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.ts
+++ b/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.ts
@@ -12,19 +12,23 @@ export function evaluatorScoresUrl(
evaluatorName: string,
evaluatorType: EvalTemplateType,
) {
+ // Code and decision-model evaluators name their scores themselves (per
+ // question for decision models), so only the evaluator ID finds them.
+ // Judge scores are named after the evaluator, and name also matches judge
+ // scores written before scores carried an evaluator ID.
const evaluatorFilter: FilterState[number] =
- evaluatorType === EvalTemplateTypeEnum.CODE
+ evaluatorType === EvalTemplateTypeEnum.LLM_AS_JUDGE
? {
- column: "evaluatorId",
+ column: "name",
type: "stringOptions",
operator: "any of",
- value: [evaluatorId],
+ value: [evaluatorName],
}
: {
- column: "name",
+ column: "evaluatorId",
type: "stringOptions",
operator: "any of",
- value: [evaluatorName],
+ value: [evaluatorId],
};
const filter: FilterState = [