Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(evals): validate decision model batch mappings (#17785)

Hassieb Pakzadb009faf54b51
diff --git a/web/src/__tests__/server/unit/batch-actions-comment-filter.servertest.ts b/web/src/__tests__/server/unit/batch-actions-comment-filter.servertest.ts
index 160dd0ffbf..ad04d2dc3b 100644
--- a/web/src/__tests__/server/unit/batch-actions-comment-filter.servertest.ts
+++ b/web/src/__tests__/server/unit/batch-actions-comment-filter.servertest.ts
@@ -95,6 +95,7 @@ function prepare({
v4BetaEnabled = false,
foundEvaluatorIds = [evaluatorId],
missingPromptVariable = false,
+ decisionModel = false,
} = {}) {
const batchActionCreate = vi
.fn()
@@ -112,29 +113,42 @@ function prepare({
if (args?.select?.id) {
return foundEvaluatorIds.map((id) => ({ id }));
}
+ const llmPrompt = missingPromptVariable
+ ? "Evaluate {{output}} {{input}}"
+ : "Evaluate {{output}}";
return foundEvaluatorIds.map((id) => ({
id,
name: "Quality",
- type: "LLM_AS_JUDGE",
+ type: decisionModel ? "DECISION_MODEL" : "LLM_AS_JUDGE",
versions: [
{
- prompt: missingPromptVariable
- ? "Evaluate {{output}} {{input}}"
- : "Evaluate {{output}}",
- promptMessages: [
- {
- role: "user",
- content: missingPromptVariable
- ? "Evaluate {{output}} {{input}}"
- : "Evaluate {{output}}",
- },
- ],
- variableMapping: [
- {
- templateVariable: "output",
- selectedColumnId: "output",
- },
- ],
+ prompt: decisionModel ? null : llmPrompt,
+ promptMessages: decisionModel
+ ? null
+ : [
+ {
+ role: "user",
+ content: llmPrompt,
+ },
+ ],
+ vars: decisionModel ? ["input", "output"] : ["output"],
+ variableMapping: decisionModel
+ ? [
+ {
+ templateVariable: "input",
+ selectedColumnId: "input",
+ },
+ {
+ templateVariable: "output",
+ selectedColumnId: "output",
+ },
+ ]
+ : [
+ {
+ templateVariable: "output",
+ selectedColumnId: "output",
+ },
+ ],
},
],
}));
@@ -438,6 +452,39 @@ describe("batched evaluation version selection", () => {
);
});
+ it("queues decision-model mappings using evaluator state variables", async () => {
+ const context = prepare({
+ v4BetaEnabled: true,
+ decisionModel: true,
+ });
+ const evaluatorMappings = [
+ {
+ evaluatorId,
+ variableMapping: [
+ { templateVariable: "input", selectedColumnId: "input" },
+ { templateVariable: "output", selectedColumnId: "output" },
+ ],
+ },
+ ];
+
+ await context.runEvaluation.create({
+ projectId,
+ query,
+ evaluatorIds: [evaluatorId],
+ sourceTable: BatchEvalSourceTable.EVENTS,
+ evalVersion: "v2",
+ evaluatorMappings,
+ });
+
+ expect(mocks.queueAdd).toHaveBeenCalledWith(
+ expect.anything(),
+ expect.objectContaining({
+ payload: expect.objectContaining({ evaluatorMappings }),
+ }),
+ expect.anything(),
+ );
+ });
+
it("rejects mapping overrides without evaluator v2", async () => {
const context = prepare({ v4BetaEnabled: true });
diff --git a/web/src/features/batch-actions/server/prepareBatchEvalEvaluatorMappings.ts b/web/src/features/batch-actions/server/prepareBatchEvalEvaluatorMappings.ts
index 624f4245d8..1a951d67fd 100644
--- a/web/src/features/batch-actions/server/prepareBatchEvalEvaluatorMappings.ts
+++ b/web/src/features/batch-actions/server/prepareBatchEvalEvaluatorMappings.ts
@@ -69,8 +69,12 @@ export async function prepareBatchEvalEvaluatorMappings(params: {
prompt: latestVersion.prompt,
promptMessages: latestVersion.promptMessages,
});
+ const requiredVariables =
+ evaluator.type === EvalTemplateType.DECISION_MODEL
+ ? latestVersion.vars
+ : extractEvaluatorPromptVariables(promptMessages);
assertCompleteEvaluatorVariableMapping({
- promptVariables: extractEvaluatorPromptVariables(promptMessages),
+ promptVariables: requiredVariables,
variableMapping:
storedVariableMapping ?? prepared.defaultVariableMapping,
});