Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(evals): track evaluator creation configuration (#16624)

Tobias Wochingerb27b66acfac8
diff --git a/web/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties.clienttest.ts b/web/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties.clienttest.ts
index 408fefaa4f..f049c59842 100644
--- a/web/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties.clienttest.ts
+++ b/web/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties.clienttest.ts
@@ -3,43 +3,57 @@ import { describe, expect, it } from "vitest";
import { getEvaluatorCreationAnalyticsProperties } from "./getEvaluatorCreationAnalyticsProperties";
describe("getEvaluatorCreationAnalyticsProperties", () => {
- it("reports which managed catalog template was used", () => {
- expect(
- getEvaluatorCreationAnalyticsProperties({
- evaluatorType: "CODE",
- creationSource: { type: "managed", templateKey: "exact-match" },
- }),
- ).toEqual({
- evaluatorType: "CODE",
- managedTemplateKey: "exact-match",
- isCustomTemplate: false,
- isFromScratch: false,
- });
- });
-
- it("reports custom templates without exposing their identity", () => {
+ it("reports all evaluator creation attributes", () => {
expect(
getEvaluatorCreationAnalyticsProperties({
evaluatorType: "LLM_AS_JUDGE",
- creationSource: { type: "custom" },
+ creationSource: { type: "managed", templateKey: "hallucination" },
+ evaluatorConfig: {
+ usesDefaultModel: false,
+ hasCustomModelParams: true,
+ scoreType: "CATEGORICAL",
+ },
+ variableMapping: [
+ {
+ templateVariable: "question",
+ selectedColumnId: "input",
+ jsonSelector: null,
+ },
+ {
+ templateVariable: "answer",
+ selectedColumnId: "output",
+ jsonSelector: "$",
+ },
+ {
+ templateVariable: "context",
+ selectedColumnId: "input",
+ jsonSelector: null,
+ },
+ ],
}),
).toEqual({
evaluatorType: "LLM_AS_JUDGE",
- isCustomTemplate: true,
+ managedTemplateKey: "hallucination",
+ isCustomTemplate: false,
isFromScratch: false,
+ usesDefaultModel: false,
+ hasCustomModelParams: true,
+ scoreType: "CATEGORICAL",
+ hasNarrowedVariableMapping: false,
+ variableMappingSources: ["input", "output"],
});
- });
- it("reports evaluators created from scratch", () => {
expect(
getEvaluatorCreationAnalyticsProperties({
- evaluatorType: "LLM_AS_JUDGE",
+ evaluatorType: "CODE",
creationSource: { type: "scratch" },
+ sourceCodeLanguage: "PYTHON",
}),
).toEqual({
- evaluatorType: "LLM_AS_JUDGE",
+ evaluatorType: "CODE",
isCustomTemplate: false,
isFromScratch: true,
+ sourceCodeLanguage: "PYTHON",
});
});
});
diff --git a/web/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties.ts b/web/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties.ts
index 6331be034b..8dc7804631 100644
--- a/web/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties.ts
+++ b/web/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties.ts
@@ -1,4 +1,8 @@
-import type { EvalTemplateType } from "@langfuse/shared";
+import type {
+ EvalTemplateSourceCodeLanguage,
+ EvalTemplateType,
+} from "@langfuse/shared";
+import type { ScoreOutputDataType } from "@/src/features/evals/v2/scoreOutputTypes";
export type EvaluatorCreationSource =
| { type: "managed"; templateKey: string }
@@ -8,16 +12,56 @@ export type EvaluatorCreationSource =
export function getEvaluatorCreationAnalyticsProperties({
evaluatorType,
creationSource,
+ evaluatorConfig,
+ sourceCodeLanguage,
+ variableMapping,
}: {
evaluatorType: EvalTemplateType;
creationSource: EvaluatorCreationSource;
+ evaluatorConfig?: {
+ usesDefaultModel: boolean;
+ hasCustomModelParams: boolean;
+ scoreType: ScoreOutputDataType;
+ };
+ sourceCodeLanguage?: EvalTemplateSourceCodeLanguage;
+ variableMapping?: Array<{
+ templateVariable?: string;
+ selectedColumnId: string | null;
+ jsonSelector?: string | null;
+ }>;
}) {
+ const configProperties = evaluatorConfig
+ ? {
+ usesDefaultModel: evaluatorConfig.usesDefaultModel,
+ hasCustomModelParams: evaluatorConfig.hasCustomModelParams,
+ scoreType: evaluatorConfig.scoreType,
+ }
+ : {};
+ const variableMappingProperties = variableMapping
+ ? {
+ hasNarrowedVariableMapping: variableMapping.some(
+ (mapping) =>
+ Boolean(mapping.jsonSelector) && mapping.jsonSelector !== "$",
+ ),
+ variableMappingSources: [
+ ...new Set(
+ variableMapping
+ .map((mapping) => mapping.selectedColumnId)
+ .filter((source): source is string => Boolean(source)),
+ ),
+ ],
+ }
+ : {};
+
if (creationSource.type === "managed") {
return {
evaluatorType,
managedTemplateKey: creationSource.templateKey,
isCustomTemplate: false,
isFromScratch: false,
+ ...configProperties,
+ ...variableMappingProperties,
+ ...(sourceCodeLanguage ? { sourceCodeLanguage } : {}),
};
}
@@ -25,5 +69,8 @@ export function getEvaluatorCreationAnalyticsProperties({
evaluatorType,
isCustomTemplate: creationSource.type === "custom",
isFromScratch: creationSource.type === "scratch",
+ ...configProperties,
+ ...variableMappingProperties,
+ ...(sourceCodeLanguage ? { sourceCodeLanguage } : {}),
};
}
diff --git a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
index eb6e205ce4..f893bdca98 100644
--- a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
+++ b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
@@ -458,6 +458,22 @@ export function EvaluatorSetupPage(
getEvaluatorCreationAnalyticsProperties({
evaluatorType: state.type,
creationSource: props.creationSource,
+ sourceCodeLanguage:
+ state.type === "CODE" ? state.sourceCodeLanguage : undefined,
+ variableMapping:
+ definition.type === "LLM_AS_JUDGE"
+ ? definition.variableMapping
+ : undefined,
+ evaluatorConfig:
+ state.type === "LLM_AS_JUDGE"
+ ? {
+ usesDefaultModel: state.modelMode === "default",
+ hasCustomModelParams:
+ state.modelMode === "custom" &&
+ Object.keys(state.modelParams ?? {}).length > 0,
+ scoreType: state.scoreOutput.dataType,
+ }
+ : undefined,
}),
);
initialSnapshot.current = getCurrentSnapshot(state);