🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(evals): track judge prompt metadata (#16880)
Tobias Wochingeradd6ca4aceb9
diff --git a/web/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties.clienttest.ts b/web/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties.clienttest.tsindex f049c59842..585ec34803 100644--- a/web/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties.clienttest.ts+++ b/web/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties.clienttest.ts@@ -1,6 +1,9 @@import { describe, expect, it } from "vitest";-import { getEvaluatorCreationAnalyticsProperties } from "./getEvaluatorCreationAnalyticsProperties";+import {+ getEvaluatorCreationAnalyticsProperties,+ getJudgePromptAnalyticsProperties,+} from "./getEvaluatorCreationAnalyticsProperties";describe("getEvaluatorCreationAnalyticsProperties", () => {it("reports all evaluator creation attributes", () => {@@ -13,6 +16,11 @@ describe("getEvaluatorCreationAnalyticsProperties", () => {hasCustomModelParams: true,scoreType: "CATEGORICAL",},+ promptMessages: [+ { role: "system" },+ { role: "user" },+ { role: "user" },+ ],variableMapping: [{templateVariable: "question",@@ -39,6 +47,8 @@ describe("getEvaluatorCreationAnalyticsProperties", () => {usesDefaultModel: false,hasCustomModelParams: true,scoreType: "CATEGORICAL",+ promptMessageCount: 3,+ promptMessageRoles: ["system", "user"],hasNarrowedVariableMapping: false,variableMappingSources: ["input", "output"],});@@ -56,4 +66,18 @@ describe("getEvaluatorCreationAnalyticsProperties", () => {sourceCodeLanguage: "PYTHON",});});++ it("reports judge prompt shape without prompt content", () => {+ expect(+ getJudgePromptAnalyticsProperties([+ { role: "system" },+ { role: "assistant" },+ { role: "user" },+ { role: "assistant" },+ ]),+ ).toEqual({+ promptMessageCount: 4,+ promptMessageRoles: ["system", "assistant", "user"],+ });+ });});diff --git a/web/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties.ts b/web/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties.tsindex 8dc7804631..f9c6ea98c4 100644--- a/web/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties.ts+++ b/web/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties.ts@@ -1,4 +1,5 @@import type {+ EvaluatorPromptMessage,EvalTemplateSourceCodeLanguage,EvalTemplateType,} from "@langfuse/shared";@@ -9,12 +10,22 @@ export type EvaluatorCreationSource =| { type: "custom" }| { type: "scratch" };+export function getJudgePromptAnalyticsProperties(+ promptMessages: Array<Pick<EvaluatorPromptMessage, "role">>,+) {+ return {+ promptMessageCount: promptMessages.length,+ promptMessageRoles: [...new Set(promptMessages.map(({ role }) => role))],+ };+}+export function getEvaluatorCreationAnalyticsProperties({evaluatorType,creationSource,evaluatorConfig,sourceCodeLanguage,variableMapping,+ promptMessages,}: {evaluatorType: EvalTemplateType;creationSource: EvaluatorCreationSource;@@ -29,6 +40,7 @@ export function getEvaluatorCreationAnalyticsProperties({selectedColumnId: string | null;jsonSelector?: string | null;}>;+ promptMessages?: Array<Pick<EvaluatorPromptMessage, "role">>;}) {const configProperties = evaluatorConfig? {@@ -52,6 +64,9 @@ export function getEvaluatorCreationAnalyticsProperties({],}: {};+ const promptProperties = promptMessages+ ? getJudgePromptAnalyticsProperties(promptMessages)+ : {};if (creationSource.type === "managed") {return {@@ -61,6 +76,7 @@ export function getEvaluatorCreationAnalyticsProperties({isFromScratch: false,...configProperties,...variableMappingProperties,+ ...promptProperties,...(sourceCodeLanguage ? { sourceCodeLanguage } : {}),};}@@ -71,6 +87,7 @@ export function getEvaluatorCreationAnalyticsProperties({isFromScratch: creationSource.type === "scratch",...configProperties,...variableMappingProperties,+ ...promptProperties,...(sourceCodeLanguage ? { sourceCodeLanguage } : {}),};}diff --git a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsxindex 37784ccf12..7ceffd2a16 100644--- a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx+++ b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx@@ -54,6 +54,7 @@ import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccesimport { useCodeEvalSourceValidation } from "@/src/features/evals/hooks/useCodeEvalSourceValidation";import {getEvaluatorCreationAnalyticsProperties,+ getJudgePromptAnalyticsProperties,type EvaluatorCreationSource,} from "@/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties";@@ -476,7 +477,12 @@ export function EvaluatorSetupPage(description,definition,});- capture("evaluators:update", { evaluatorType: state.type });+ capture("evaluators:update", {+ evaluatorType: state.type,+ ...(definition.type === "LLM_AS_JUDGE"+ ? getJudgePromptAnalyticsProperties(definition.promptMessages)+ : {}),+ });showSuccessToast({title: "Evaluator saved",description: "Your evaluator changes are saved.",@@ -505,6 +511,10 @@ export function EvaluatorSetupPage(definition.type === "LLM_AS_JUDGE"? definition.variableMapping: undefined,+ promptMessages:+ definition.type === "LLM_AS_JUDGE"+ ? definition.promptMessages+ : undefined,evaluatorConfig:state.type === "LLM_AS_JUDGE"? {