🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(evals): track evaluator creation configuration (#16624)
Tobias Wochingerb27b66acfac8
diff --git a/web/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties.clienttest.ts b/web/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties.clienttest.tsindex 408fefaa4f..f049c59842 100644--- a/web/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties.clienttest.ts+++ b/web/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties.clienttest.ts@@ -3,43 +3,57 @@ import { describe, expect, it } from "vitest";import { getEvaluatorCreationAnalyticsProperties } from "./getEvaluatorCreationAnalyticsProperties";describe("getEvaluatorCreationAnalyticsProperties", () => {- it("reports which managed catalog template was used", () => {- expect(- getEvaluatorCreationAnalyticsProperties({- evaluatorType: "CODE",- creationSource: { type: "managed", templateKey: "exact-match" },- }),- ).toEqual({- evaluatorType: "CODE",- managedTemplateKey: "exact-match",- isCustomTemplate: false,- isFromScratch: false,- });- });-- it("reports custom templates without exposing their identity", () => {+ it("reports all evaluator creation attributes", () => {expect(getEvaluatorCreationAnalyticsProperties({evaluatorType: "LLM_AS_JUDGE",- creationSource: { type: "custom" },+ creationSource: { type: "managed", templateKey: "hallucination" },+ evaluatorConfig: {+ usesDefaultModel: false,+ hasCustomModelParams: true,+ scoreType: "CATEGORICAL",+ },+ variableMapping: [+ {+ templateVariable: "question",+ selectedColumnId: "input",+ jsonSelector: null,+ },+ {+ templateVariable: "answer",+ selectedColumnId: "output",+ jsonSelector: "$",+ },+ {+ templateVariable: "context",+ selectedColumnId: "input",+ jsonSelector: null,+ },+ ],}),).toEqual({evaluatorType: "LLM_AS_JUDGE",- isCustomTemplate: true,+ managedTemplateKey: "hallucination",+ isCustomTemplate: false,isFromScratch: false,+ usesDefaultModel: false,+ hasCustomModelParams: true,+ scoreType: "CATEGORICAL",+ hasNarrowedVariableMapping: false,+ variableMappingSources: ["input", "output"],});- });- it("reports evaluators created from scratch", () => {expect(getEvaluatorCreationAnalyticsProperties({- evaluatorType: "LLM_AS_JUDGE",+ evaluatorType: "CODE",creationSource: { type: "scratch" },+ sourceCodeLanguage: "PYTHON",}),).toEqual({- evaluatorType: "LLM_AS_JUDGE",+ evaluatorType: "CODE",isCustomTemplate: false,isFromScratch: true,+ sourceCodeLanguage: "PYTHON",});});});diff --git a/web/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties.ts b/web/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties.tsindex 6331be034b..8dc7804631 100644--- a/web/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties.ts+++ b/web/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties.ts@@ -1,4 +1,8 @@-import type { EvalTemplateType } from "@langfuse/shared";+import type {+ EvalTemplateSourceCodeLanguage,+ EvalTemplateType,+} from "@langfuse/shared";+import type { ScoreOutputDataType } from "@/src/features/evals/v2/scoreOutputTypes";export type EvaluatorCreationSource =| { type: "managed"; templateKey: string }@@ -8,16 +12,56 @@ export type EvaluatorCreationSource =export function getEvaluatorCreationAnalyticsProperties({evaluatorType,creationSource,+ evaluatorConfig,+ sourceCodeLanguage,+ variableMapping,}: {evaluatorType: EvalTemplateType;creationSource: EvaluatorCreationSource;+ evaluatorConfig?: {+ usesDefaultModel: boolean;+ hasCustomModelParams: boolean;+ scoreType: ScoreOutputDataType;+ };+ sourceCodeLanguage?: EvalTemplateSourceCodeLanguage;+ variableMapping?: Array<{+ templateVariable?: string;+ selectedColumnId: string | null;+ jsonSelector?: string | null;+ }>;}) {+ const configProperties = evaluatorConfig+ ? {+ usesDefaultModel: evaluatorConfig.usesDefaultModel,+ hasCustomModelParams: evaluatorConfig.hasCustomModelParams,+ scoreType: evaluatorConfig.scoreType,+ }+ : {};+ const variableMappingProperties = variableMapping+ ? {+ hasNarrowedVariableMapping: variableMapping.some(+ (mapping) =>+ Boolean(mapping.jsonSelector) && mapping.jsonSelector !== "$",+ ),+ variableMappingSources: [+ ...new Set(+ variableMapping+ .map((mapping) => mapping.selectedColumnId)+ .filter((source): source is string => Boolean(source)),+ ),+ ],+ }+ : {};+if (creationSource.type === "managed") {return {evaluatorType,managedTemplateKey: creationSource.templateKey,isCustomTemplate: false,isFromScratch: false,+ ...configProperties,+ ...variableMappingProperties,+ ...(sourceCodeLanguage ? { sourceCodeLanguage } : {}),};}@@ -25,5 +69,8 @@ export function getEvaluatorCreationAnalyticsProperties({evaluatorType,isCustomTemplate: creationSource.type === "custom",isFromScratch: creationSource.type === "scratch",+ ...configProperties,+ ...variableMappingProperties,+ ...(sourceCodeLanguage ? { sourceCodeLanguage } : {}),};}diff --git a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsxindex eb6e205ce4..f893bdca98 100644--- a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx+++ b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx@@ -458,6 +458,22 @@ export function EvaluatorSetupPage(getEvaluatorCreationAnalyticsProperties({evaluatorType: state.type,creationSource: props.creationSource,+ sourceCodeLanguage:+ state.type === "CODE" ? state.sourceCodeLanguage : undefined,+ variableMapping:+ definition.type === "LLM_AS_JUDGE"+ ? definition.variableMapping+ : undefined,+ evaluatorConfig:+ state.type === "LLM_AS_JUDGE"+ ? {+ usesDefaultModel: state.modelMode === "default",+ hasCustomModelParams:+ state.modelMode === "custom" &&+ Object.keys(state.modelParams ?? {}).length > 0,+ scoreType: state.scoreOutput.dataType,+ }+ : undefined,}),);initialSnapshot.current = getCurrentSnapshot(state);