🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(evals): stop validating rule mappings when updating an evaluator (#16460)
cursor[bot]cb823daddc00
diff --git a/web/src/__tests__/server/evaluator-v2-service.servertest.ts b/web/src/__tests__/server/evaluator-v2-service.servertest.tsindex 23ec7d9550..6030b68cf1 100644--- a/web/src/__tests__/server/evaluator-v2-service.servertest.ts+++ b/web/src/__tests__/server/evaluator-v2-service.servertest.ts@@ -588,11 +588,18 @@ describe("EvaluatorService", () => {).rejects.toThrow("Evaluator not found");});- it("rejects prompt changes that make existing rule mappings incomplete", async () => {+ it("changes the prompt without validating or rewriting rule mappings", async () => {const service = createService();const input = llmInput("Assigned evaluator update");const created = await service.create(input, null);- await prisma.evaluationRule.create({+ const staleMapping = [+ { templateVariable: "output", selectedColumnId: "output" },+ {+ templateVariable: "item_metadata",+ selectedColumnId: "experimentItemMetadata",+ },+ ];+ const rule = await prisma.evaluationRule.create({data: {projectId,name: "Assigned rule",@@ -605,19 +612,18 @@ describe("EvaluatorService", () => {create: {projectId,evaluatorId: created.id,- variableMapping: input.definition- .variableMapping as Prisma.InputJsonValue,+ variableMapping: staleMapping as Prisma.InputJsonValue,},},},});+ // The override maps a variable the new prompt drops and misses one it adds.await expect(service.update({...input,evaluatorId: created.id,- name: "Invalid renamed evaluator",definition: {...input.definition,prompt: "Judge {{input}} and {{output}}",@@ -630,12 +636,15 @@ describe("EvaluatorService", () => {},null,),- ).rejects.toThrow("Missing mappings for evaluator variables: input");-- await expect(service.get(projectId, created.id)).resolves.toMatchObject({- name: input.name,- versions: [expect.objectContaining({ version: 1 })],+ ).resolves.toMatchObject({+ versions: [expect.objectContaining({ version: 2 })],});++ const assignment =+ await prisma.evaluationRuleEvaluatorAssignment.findFirstOrThrow({+ where: { evaluationRuleId: rule.id, evaluatorId: created.id },+ });+ expect(assignment.variableMapping).toEqual(staleMapping);});it("returns a retryable conflict when an evaluator version advances concurrently", async () => {diff --git a/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.ts b/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.tsindex 982b4571ee..26fac4f3a4 100644--- a/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.ts+++ b/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.ts@@ -152,6 +152,38 @@ describe("evaluator configuration validation", () => {).rejects.toThrow("Evaluator variables must match the prompt variables");});+ it("rejects mappings that reference variables not in the prompt", async () => {+ await expect(+ assertEvaluatorConfigurationValid({+ projectId: "project-id",+ name: "LLM evaluator",+ definition: {+ type: EvalTemplateType.LLM_AS_JUDGE,+ prompt: "Judge {{output}}",+ provider: null,+ model: null,+ modelParams: null,+ vars: ["output"],+ variableMapping: [+ { templateVariable: "output", selectedColumnId: "output" },+ {+ templateVariable: "item_metadata",+ selectedColumnId: "experimentItemMetadata",+ },+ ],+ outputDefinition: {+ version: 2,+ dataType: "NUMERIC",+ score: { description: "Quality" },+ reasoning: { description: "Reasoning" },+ },+ },+ }),+ ).rejects.toThrow(+ "Mappings reference unknown evaluator variables: item_metadata",+ );+ });+it("rejects incomplete evaluator default mappings", async () => {await expect(assertEvaluatorConfigurationValid({diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts b/web/src/features/evals/v2/server/evaluators/evaluatorRepository.tsindex 35c3706f6a..6223a1c294 100644--- a/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts+++ b/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts@@ -473,23 +473,6 @@ export function updateEvaluatorMetadata(params: {});}-export function findRuleMappingOverridesForEvaluator(params: {- prisma: PrismaClient | PrismaTransaction;- projectId: string;- evaluatorId: string;-}) {- return params.prisma.evaluationRuleEvaluatorAssignment.findMany({- where: {- projectId: params.projectId,- evaluatorId: params.evaluatorId,- evaluationRule: {- targetObject: { in: ["event", "experiment"] },- },- },- select: { variableMapping: true },- });-}-export async function appendEvaluatorVersion(params: {tx: PrismaTransaction;evaluatorId: string;diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorService.ts b/web/src/features/evals/v2/server/evaluators/evaluatorService.tsindex 039535be8f..dcb9c01c53 100644--- a/web/src/features/evals/v2/server/evaluators/evaluatorService.ts+++ b/web/src/features/evals/v2/server/evaluators/evaluatorService.ts@@ -39,10 +39,7 @@ import {EvaluatorModelConfigurationError,EvaluatorVersionConflictError,} from "./evaluatorErrors";-import {- assertCompleteEvaluatorVariableMapping,- assertEvaluatorConfigurationValid,-} from "./evaluatorValidation";+import { assertEvaluatorConfigurationValid } from "./evaluatorValidation";type SuggestEvaluatorTextParams = {projectId: string;@@ -587,24 +584,6 @@ async function updateEvaluator(params: {input.definition,);- if (- input.definition.type === EvalTemplateType.LLM_AS_JUDGE &&- (definitionChanged || params.forceNewVersion)- ) {- const assignments = await repository.findRuleMappingOverridesForEvaluator({- prisma: tx,- projectId: input.projectId,- evaluatorId: input.evaluatorId,- });- for (const assignment of assignments) {- assertCompleteEvaluatorVariableMapping({- prompt: input.definition.prompt,- variableMapping:- assignment.variableMapping ?? input.definition.variableMapping ?? [],- });- }- }-await repository.updateEvaluatorMetadata({tx,projectId: input.projectId,