🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(evals): validate decision model batch mappings (#17785)
Hassieb Pakzadb009faf54b51
diff --git a/web/src/__tests__/server/unit/batch-actions-comment-filter.servertest.ts b/web/src/__tests__/server/unit/batch-actions-comment-filter.servertest.tsindex 160dd0ffbf..ad04d2dc3b 100644--- a/web/src/__tests__/server/unit/batch-actions-comment-filter.servertest.ts+++ b/web/src/__tests__/server/unit/batch-actions-comment-filter.servertest.ts@@ -95,6 +95,7 @@ function prepare({v4BetaEnabled = false,foundEvaluatorIds = [evaluatorId],missingPromptVariable = false,+ decisionModel = false,} = {}) {const batchActionCreate = vi.fn()@@ -112,29 +113,42 @@ function prepare({if (args?.select?.id) {return foundEvaluatorIds.map((id) => ({ id }));}+ const llmPrompt = missingPromptVariable+ ? "Evaluate {{output}} {{input}}"+ : "Evaluate {{output}}";return foundEvaluatorIds.map((id) => ({id,name: "Quality",- type: "LLM_AS_JUDGE",+ type: decisionModel ? "DECISION_MODEL" : "LLM_AS_JUDGE",versions: [{- prompt: missingPromptVariable- ? "Evaluate {{output}} {{input}}"- : "Evaluate {{output}}",- promptMessages: [- {- role: "user",- content: missingPromptVariable- ? "Evaluate {{output}} {{input}}"- : "Evaluate {{output}}",- },- ],- variableMapping: [- {- templateVariable: "output",- selectedColumnId: "output",- },- ],+ prompt: decisionModel ? null : llmPrompt,+ promptMessages: decisionModel+ ? null+ : [+ {+ role: "user",+ content: llmPrompt,+ },+ ],+ vars: decisionModel ? ["input", "output"] : ["output"],+ variableMapping: decisionModel+ ? [+ {+ templateVariable: "input",+ selectedColumnId: "input",+ },+ {+ templateVariable: "output",+ selectedColumnId: "output",+ },+ ]+ : [+ {+ templateVariable: "output",+ selectedColumnId: "output",+ },+ ],},],}));@@ -438,6 +452,39 @@ describe("batched evaluation version selection", () => {);});+ it("queues decision-model mappings using evaluator state variables", async () => {+ const context = prepare({+ v4BetaEnabled: true,+ decisionModel: true,+ });+ const evaluatorMappings = [+ {+ evaluatorId,+ variableMapping: [+ { templateVariable: "input", selectedColumnId: "input" },+ { templateVariable: "output", selectedColumnId: "output" },+ ],+ },+ ];++ await context.runEvaluation.create({+ projectId,+ query,+ evaluatorIds: [evaluatorId],+ sourceTable: BatchEvalSourceTable.EVENTS,+ evalVersion: "v2",+ evaluatorMappings,+ });++ expect(mocks.queueAdd).toHaveBeenCalledWith(+ expect.anything(),+ expect.objectContaining({+ payload: expect.objectContaining({ evaluatorMappings }),+ }),+ expect.anything(),+ );+ });+it("rejects mapping overrides without evaluator v2", async () => {const context = prepare({ v4BetaEnabled: true });diff --git a/web/src/features/batch-actions/server/prepareBatchEvalEvaluatorMappings.ts b/web/src/features/batch-actions/server/prepareBatchEvalEvaluatorMappings.tsindex 624f4245d8..1a951d67fd 100644--- a/web/src/features/batch-actions/server/prepareBatchEvalEvaluatorMappings.ts+++ b/web/src/features/batch-actions/server/prepareBatchEvalEvaluatorMappings.ts@@ -69,8 +69,12 @@ export async function prepareBatchEvalEvaluatorMappings(params: {prompt: latestVersion.prompt,promptMessages: latestVersion.promptMessages,});+ const requiredVariables =+ evaluator.type === EvalTemplateType.DECISION_MODEL+ ? latestVersion.vars+ : extractEvaluatorPromptVariables(promptMessages);assertCompleteEvaluatorVariableMapping({- promptVariables: extractEvaluatorPromptVariables(promptMessages),+ promptVariables: requiredVariables,variableMapping:storedVariableMapping ?? prepared.defaultVariableMapping,});