Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(evals): stop validating rule mappings when updating an evaluator (#16460)

cursor[bot]cb823daddc00
diff --git a/web/src/__tests__/server/evaluator-v2-service.servertest.ts b/web/src/__tests__/server/evaluator-v2-service.servertest.ts
index 23ec7d9550..6030b68cf1 100644
--- a/web/src/__tests__/server/evaluator-v2-service.servertest.ts
+++ b/web/src/__tests__/server/evaluator-v2-service.servertest.ts
@@ -588,11 +588,18 @@ describe("EvaluatorService", () => {
).rejects.toThrow("Evaluator not found");
});
- it("rejects prompt changes that make existing rule mappings incomplete", async () => {
+ it("changes the prompt without validating or rewriting rule mappings", async () => {
const service = createService();
const input = llmInput("Assigned evaluator update");
const created = await service.create(input, null);
- await prisma.evaluationRule.create({
+ const staleMapping = [
+ { templateVariable: "output", selectedColumnId: "output" },
+ {
+ templateVariable: "item_metadata",
+ selectedColumnId: "experimentItemMetadata",
+ },
+ ];
+ const rule = await prisma.evaluationRule.create({
data: {
projectId,
name: "Assigned rule",
@@ -605,19 +612,18 @@ describe("EvaluatorService", () => {
create: {
projectId,
evaluatorId: created.id,
- variableMapping: input.definition
- .variableMapping as Prisma.InputJsonValue,
+ variableMapping: staleMapping as Prisma.InputJsonValue,
},
},
},
});
+ // The override maps a variable the new prompt drops and misses one it adds.
await expect(
service.update(
{
...input,
evaluatorId: created.id,
- name: "Invalid renamed evaluator",
definition: {
...input.definition,
prompt: "Judge {{input}} and {{output}}",
@@ -630,12 +636,15 @@ describe("EvaluatorService", () => {
},
null,
),
- ).rejects.toThrow("Missing mappings for evaluator variables: input");
-
- await expect(service.get(projectId, created.id)).resolves.toMatchObject({
- name: input.name,
- versions: [expect.objectContaining({ version: 1 })],
+ ).resolves.toMatchObject({
+ versions: [expect.objectContaining({ version: 2 })],
});
+
+ const assignment =
+ await prisma.evaluationRuleEvaluatorAssignment.findFirstOrThrow({
+ where: { evaluationRuleId: rule.id, evaluatorId: created.id },
+ });
+ expect(assignment.variableMapping).toEqual(staleMapping);
});
it("returns a retryable conflict when an evaluator version advances concurrently", async () => {
diff --git a/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.ts b/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.ts
index 982b4571ee..26fac4f3a4 100644
--- a/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.ts
+++ b/web/src/__tests__/server/unit/evaluator-v2-validation.servertest.ts
@@ -152,6 +152,38 @@ describe("evaluator configuration validation", () => {
).rejects.toThrow("Evaluator variables must match the prompt variables");
});
+ it("rejects mappings that reference variables not in the prompt", async () => {
+ await expect(
+ assertEvaluatorConfigurationValid({
+ projectId: "project-id",
+ name: "LLM evaluator",
+ definition: {
+ type: EvalTemplateType.LLM_AS_JUDGE,
+ prompt: "Judge {{output}}",
+ provider: null,
+ model: null,
+ modelParams: null,
+ vars: ["output"],
+ variableMapping: [
+ { templateVariable: "output", selectedColumnId: "output" },
+ {
+ templateVariable: "item_metadata",
+ selectedColumnId: "experimentItemMetadata",
+ },
+ ],
+ outputDefinition: {
+ version: 2,
+ dataType: "NUMERIC",
+ score: { description: "Quality" },
+ reasoning: { description: "Reasoning" },
+ },
+ },
+ }),
+ ).rejects.toThrow(
+ "Mappings reference unknown evaluator variables: item_metadata",
+ );
+ });
+
it("rejects incomplete evaluator default mappings", async () => {
await expect(
assertEvaluatorConfigurationValid({
diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts b/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts
index 35c3706f6a..6223a1c294 100644
--- a/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts
+++ b/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts
@@ -473,23 +473,6 @@ export function updateEvaluatorMetadata(params: {
});
}
-export function findRuleMappingOverridesForEvaluator(params: {
- prisma: PrismaClient | PrismaTransaction;
- projectId: string;
- evaluatorId: string;
-}) {
- return params.prisma.evaluationRuleEvaluatorAssignment.findMany({
- where: {
- projectId: params.projectId,
- evaluatorId: params.evaluatorId,
- evaluationRule: {
- targetObject: { in: ["event", "experiment"] },
- },
- },
- select: { variableMapping: true },
- });
-}
-
export async function appendEvaluatorVersion(params: {
tx: PrismaTransaction;
evaluatorId: string;
diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorService.ts b/web/src/features/evals/v2/server/evaluators/evaluatorService.ts
index 039535be8f..dcb9c01c53 100644
--- a/web/src/features/evals/v2/server/evaluators/evaluatorService.ts
+++ b/web/src/features/evals/v2/server/evaluators/evaluatorService.ts
@@ -39,10 +39,7 @@ import {
EvaluatorModelConfigurationError,
EvaluatorVersionConflictError,
} from "./evaluatorErrors";
-import {
- assertCompleteEvaluatorVariableMapping,
- assertEvaluatorConfigurationValid,
-} from "./evaluatorValidation";
+import { assertEvaluatorConfigurationValid } from "./evaluatorValidation";
type SuggestEvaluatorTextParams = {
projectId: string;
@@ -587,24 +584,6 @@ async function updateEvaluator(params: {
input.definition,
);
- if (
- input.definition.type === EvalTemplateType.LLM_AS_JUDGE &&
- (definitionChanged || params.forceNewVersion)
- ) {
- const assignments = await repository.findRuleMappingOverridesForEvaluator({
- prisma: tx,
- projectId: input.projectId,
- evaluatorId: input.evaluatorId,
- });
- for (const assignment of assignments) {
- assertCompleteEvaluatorVariableMapping({
- prompt: input.definition.prompt,
- variableMapping:
- assignment.variableMapping ?? input.definition.variableMapping ?? [],
- });
- }
- }
-
await repository.updateEvaluatorMetadata({
tx,
projectId: input.projectId,