Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(evals): track judge prompt metadata (#16880)

Tobias Wochingeradd6ca4aceb9
diff --git a/web/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties.clienttest.ts b/web/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties.clienttest.ts
index f049c59842..585ec34803 100644
--- a/web/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties.clienttest.ts
+++ b/web/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties.clienttest.ts
@@ -1,6 +1,9 @@
import { describe, expect, it } from "vitest";
-import { getEvaluatorCreationAnalyticsProperties } from "./getEvaluatorCreationAnalyticsProperties";
+import {
+ getEvaluatorCreationAnalyticsProperties,
+ getJudgePromptAnalyticsProperties,
+} from "./getEvaluatorCreationAnalyticsProperties";
describe("getEvaluatorCreationAnalyticsProperties", () => {
it("reports all evaluator creation attributes", () => {
@@ -13,6 +16,11 @@ describe("getEvaluatorCreationAnalyticsProperties", () => {
hasCustomModelParams: true,
scoreType: "CATEGORICAL",
},
+ promptMessages: [
+ { role: "system" },
+ { role: "user" },
+ { role: "user" },
+ ],
variableMapping: [
{
templateVariable: "question",
@@ -39,6 +47,8 @@ describe("getEvaluatorCreationAnalyticsProperties", () => {
usesDefaultModel: false,
hasCustomModelParams: true,
scoreType: "CATEGORICAL",
+ promptMessageCount: 3,
+ promptMessageRoles: ["system", "user"],
hasNarrowedVariableMapping: false,
variableMappingSources: ["input", "output"],
});
@@ -56,4 +66,18 @@ describe("getEvaluatorCreationAnalyticsProperties", () => {
sourceCodeLanguage: "PYTHON",
});
});
+
+ it("reports judge prompt shape without prompt content", () => {
+ expect(
+ getJudgePromptAnalyticsProperties([
+ { role: "system" },
+ { role: "assistant" },
+ { role: "user" },
+ { role: "assistant" },
+ ]),
+ ).toEqual({
+ promptMessageCount: 4,
+ promptMessageRoles: ["system", "assistant", "user"],
+ });
+ });
});
diff --git a/web/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties.ts b/web/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties.ts
index 8dc7804631..f9c6ea98c4 100644
--- a/web/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties.ts
+++ b/web/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties.ts
@@ -1,4 +1,5 @@
import type {
+ EvaluatorPromptMessage,
EvalTemplateSourceCodeLanguage,
EvalTemplateType,
} from "@langfuse/shared";
@@ -9,12 +10,22 @@ export type EvaluatorCreationSource =
| { type: "custom" }
| { type: "scratch" };
+export function getJudgePromptAnalyticsProperties(
+ promptMessages: Array<Pick<EvaluatorPromptMessage, "role">>,
+) {
+ return {
+ promptMessageCount: promptMessages.length,
+ promptMessageRoles: [...new Set(promptMessages.map(({ role }) => role))],
+ };
+}
+
export function getEvaluatorCreationAnalyticsProperties({
evaluatorType,
creationSource,
evaluatorConfig,
sourceCodeLanguage,
variableMapping,
+ promptMessages,
}: {
evaluatorType: EvalTemplateType;
creationSource: EvaluatorCreationSource;
@@ -29,6 +40,7 @@ export function getEvaluatorCreationAnalyticsProperties({
selectedColumnId: string | null;
jsonSelector?: string | null;
}>;
+ promptMessages?: Array<Pick<EvaluatorPromptMessage, "role">>;
}) {
const configProperties = evaluatorConfig
? {
@@ -52,6 +64,9 @@ export function getEvaluatorCreationAnalyticsProperties({
],
}
: {};
+ const promptProperties = promptMessages
+ ? getJudgePromptAnalyticsProperties(promptMessages)
+ : {};
if (creationSource.type === "managed") {
return {
@@ -61,6 +76,7 @@ export function getEvaluatorCreationAnalyticsProperties({
isFromScratch: false,
...configProperties,
...variableMappingProperties,
+ ...promptProperties,
...(sourceCodeLanguage ? { sourceCodeLanguage } : {}),
};
}
@@ -71,6 +87,7 @@ export function getEvaluatorCreationAnalyticsProperties({
isFromScratch: creationSource.type === "scratch",
...configProperties,
...variableMappingProperties,
+ ...promptProperties,
...(sourceCodeLanguage ? { sourceCodeLanguage } : {}),
};
}
diff --git a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
index 37784ccf12..7ceffd2a16 100644
--- a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
+++ b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
@@ -54,6 +54,7 @@ import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAcces
import { useCodeEvalSourceValidation } from "@/src/features/evals/hooks/useCodeEvalSourceValidation";
import {
getEvaluatorCreationAnalyticsProperties,
+ getJudgePromptAnalyticsProperties,
type EvaluatorCreationSource,
} from "@/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties";
@@ -476,7 +477,12 @@ export function EvaluatorSetupPage(
description,
definition,
});
- capture("evaluators:update", { evaluatorType: state.type });
+ capture("evaluators:update", {
+ evaluatorType: state.type,
+ ...(definition.type === "LLM_AS_JUDGE"
+ ? getJudgePromptAnalyticsProperties(definition.promptMessages)
+ : {}),
+ });
showSuccessToast({
title: "Evaluator saved",
description: "Your evaluator changes are saved.",
@@ -505,6 +511,10 @@ export function EvaluatorSetupPage(
definition.type === "LLM_AS_JUDGE"
? definition.variableMapping
: undefined,
+ promptMessages:
+ definition.type === "LLM_AS_JUDGE"
+ ? definition.promptMessages
+ : undefined,
evaluatorConfig:
state.type === "LLM_AS_JUDGE"
? {