Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(evals): link code evaluators to their scores (#16438)

Tobias Wochingerc23526cbee5c
diff --git a/packages/shared/src/server/tableMappings/mapScoresTable.ts b/packages/shared/src/server/tableMappings/mapScoresTable.ts
index 804c4cc921..cd2ddbb7a6 100644
--- a/packages/shared/src/server/tableMappings/mapScoresTable.ts
+++ b/packages/shared/src/server/tableMappings/mapScoresTable.ts
@@ -59,6 +59,13 @@ export const scoresTableUiColumnDefinitions: UiColumnMappings = [
clickhouseSelect: "name",
queryPrefix: "s",
},
+ {
+ uiTableName: "Evaluator ID",
+ uiTableId: "evaluatorId",
+ clickhouseTableName: "scores",
+ clickhouseSelect:
+ "coalesce(nullIf(s.metadata['evaluator_id'], ''), s.metadata['job_configuration_id'])",
+ },
{
uiTableName: "Value",
uiTableId: "value",
diff --git a/packages/shared/src/tableDefinitions/scoresTable.ts b/packages/shared/src/tableDefinitions/scoresTable.ts
index 02d7341d41..6a09daaac5 100644
--- a/packages/shared/src/tableDefinitions/scoresTable.ts
+++ b/packages/shared/src/tableDefinitions/scoresTable.ts
@@ -62,6 +62,14 @@ export const scoresTableCols: ColumnDefinition[] = [
internal: 's."name"',
options: [],
},
+ {
+ name: "Evaluator ID",
+ id: "evaluatorId",
+ type: "stringOptions",
+ // ClickHouse-only virtual column; see mapScoresTable.ts.
+ internal: "",
+ options: [],
+ },
{ name: "Value", id: "value", type: "number", internal: 's."value"' },
{
name: "Boolean Value",
diff --git a/web/src/__tests__/server/scores-trpc.servertest.ts b/web/src/__tests__/server/scores-trpc.servertest.ts
index b09d4ec991..c6238d4b77 100644
--- a/web/src/__tests__/server/scores-trpc.servertest.ts
+++ b/web/src/__tests__/server/scores-trpc.servertest.ts
@@ -195,6 +195,76 @@ describe("scores trpc", () => {
);
});
+ it("filters evaluator scores by recorded evaluator and legacy rule metadata", async () => {
+ const [evaluator, otherEvaluator] = await Promise.all(
+ ["Evaluator", "Other evaluator"].map((name) =>
+ prisma.evaluator.create({
+ data: { projectId, name, type: "CODE" },
+ }),
+ ),
+ );
+ const rule = await prisma.evaluationRule.create({
+ data: {
+ projectId,
+ name: "Legacy rule",
+ targetObject: "EVENT",
+ filter: [],
+ sampling: 1,
+ delay: 0,
+ assignments: {
+ create: { projectId, evaluatorId: evaluator.id },
+ },
+ },
+ });
+ const directScore = createTraceScore({
+ project_id: projectId,
+ name: "direct-score",
+ metadata: { evaluator_id: evaluator.id },
+ });
+ const legacyScore = createTraceScore({
+ project_id: projectId,
+ name: "legacy-score-with-a-different-name",
+ metadata: { job_configuration_id: rule.id },
+ });
+ const otherScore = createTraceScore({
+ project_id: projectId,
+ name: "other-score",
+ metadata: { evaluator_id: otherEvaluator.id },
+ });
+ await createScoresCh([directScore, legacyScore, otherScore]);
+
+ const payload = {
+ projectId,
+ filter: [
+ {
+ column: "evaluatorId",
+ type: "stringOptions" as const,
+ operator: "any of" as const,
+ value: [evaluator.id, rule.id],
+ },
+ ],
+ orderBy: { column: "timestamp", order: "DESC" as const },
+ page: 0,
+ limit: 50,
+ };
+
+ const [scores, eventScores, count, eventCount] = await Promise.all([
+ caller.scores.all(payload),
+ caller.scores.allFromEvents(payload),
+ caller.scores.countAll({ ...payload, orderBy: null }),
+ caller.scores.countAllFromEvents({ ...payload, orderBy: null }),
+ ]);
+
+ expect(new Set(scores.scores.map(({ id }) => id))).toEqual(
+ new Set([directScore.id, legacyScore.id]),
+ );
+ expect(new Set(eventScores.scores.map(({ id }) => id))).toEqual(
+ new Set([directScore.id, legacyScore.id]),
+ );
+ expect(count.totalCount).toBe(2);
+ expect(eventCount.totalCount).toBe(2);
+ });
+
it("does not match empty boolean representations when filtering boolean values", async () => {
const trueBooleanScore = createTraceScore({
project_id: projectId,
diff --git a/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.clienttest.ts b/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.clienttest.ts
index 0f953fdc48..e54ececc06 100644
--- a/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.clienttest.ts
+++ b/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.clienttest.ts
@@ -11,20 +11,54 @@ import {
} from "./evaluatorScoresUrl";
describe("evaluatorScoresUrl", () => {
- it("shows evaluator scores across all environments", () => {
+ it("filters code evaluator scores by evaluator and assigned rule IDs", () => {
const url = new URL(
- evaluatorScoresUrl("project-1", "evaluator-name"),
+ evaluatorScoresUrl(
+ "project-1",
+ "evaluator-id",
+ "Code evaluator",
+ EvalTemplateTypeEnum.CODE,
+ ["rule-1", "rule-2"],
+ ),
"https://langfuse.local",
);
expect(url.pathname).toBe("/project/project-1/scores");
expect(url.searchParams.get("showAllEnvironments")).toBe("true");
+ expect(decodeFiltersGeneric(url.searchParams.get("filter") ?? "")).toEqual([
+ {
+ column: "evaluatorId",
+ type: "stringOptions",
+ operator: "any of",
+ value: ["evaluator-id", "rule-1", "rule-2"],
+ },
+ {
+ column: "source",
+ type: "stringOptions",
+ operator: "any of",
+ value: ["EVAL"],
+ },
+ ]);
+ });
+
+ it("filters judge scores by evaluator name", () => {
+ const url = new URL(
+ evaluatorScoresUrl(
+ "project-1",
+ "evaluator-id",
+ "Correctness",
+ EvalTemplateTypeEnum.LLM_AS_JUDGE,
+ ["rule-1"],
+ ),
+ "https://langfuse.local",
+ );
+
expect(decodeFiltersGeneric(url.searchParams.get("filter") ?? "")).toEqual([
{
column: "name",
type: "stringOptions",
operator: "any of",
- value: ["evaluator-name"],
+ value: ["Correctness"],
},
{
column: "source",
diff --git a/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.ts b/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.ts
index 683d79cca7..8116310b46 100644
--- a/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.ts
+++ b/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.ts
@@ -6,14 +6,30 @@ import {
encodeFiltersGeneric,
} from "@langfuse/shared";
-export function evaluatorScoresUrl(projectId: string, name: string) {
+export function evaluatorScoresUrl(
+ projectId: string,
+ evaluatorId: string,
+ evaluatorName: string,
+ evaluatorType: EvalTemplateType,
+ assignedRuleIds: string[],
+) {
+ const evaluatorFilter: FilterState[number] =
+ evaluatorType === EvalTemplateTypeEnum.CODE
+ ? {
+ column: "evaluatorId",
+ type: "stringOptions",
+ operator: "any of",
+ value: [...new Set([evaluatorId, ...assignedRuleIds])],
+ }
+ : {
+ column: "name",
+ type: "stringOptions",
+ operator: "any of",
+ value: [evaluatorName],
+ };
+
const filter: FilterState = [
- {
- column: "name",
- type: "stringOptions",
- operator: "any of",
- value: [name],
- },
+ evaluatorFilter,
{
column: "source",
type: "stringOptions",
diff --git a/web/src/features/evals/v2/pages/EvaluatorsPage.tsx b/web/src/features/evals/v2/pages/EvaluatorsPage.tsx
index adfdb11aeb..11ea4d8ee0 100644
--- a/web/src/features/evals/v2/pages/EvaluatorsPage.tsx
+++ b/web/src/features/evals/v2/pages/EvaluatorsPage.tsx
@@ -443,7 +443,15 @@ export default function EvaluatorsPage() {
<EvaluatorActionsCell
canViewExecutions={hasExecutionReadAccess}
onViewScores={() =>
- router.push(evaluatorScoresUrl(projectId, row.original.name))
+ router.push(
+ evaluatorScoresUrl(
+ projectId,
+ row.original.id,
+ row.original.name,
+ row.original.type,
+ row.original.assignedRuleIds,
+ ),
+ )
}
onViewExecutions={() =>
router.push(
diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts b/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts
index 7391fa9b58..35c3706f6a 100644
--- a/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts
+++ b/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts
@@ -180,7 +180,7 @@ export async function listEvaluators(params: {
assignments: {
where: { projectId: params.projectId },
select: {
- evaluationRule: { select: { status: true } },
+ evaluationRule: { select: { id: true, status: true } },
},
},
},
@@ -190,6 +190,9 @@ export async function listEvaluators(params: {
return {
evaluators: evaluators.map(({ assignments, ...evaluator }) => ({
...evaluator,
+ assignedRuleIds: assignments.map(
+ ({ evaluationRule }) => evaluationRule.id,
+ ),
hasActiveRules: assignments.some(
({ evaluationRule }) => evaluationRule.status === "ACTIVE",
),