Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(evals): pass experiment context to v2 code evaluator test runs (#19167)

Tobias Wochinger61937211f01a
diff --git a/web/src/__tests__/server/unit/evaluator-v2-test-evaluator.servertest.ts b/web/src/__tests__/server/unit/evaluator-v2-test-evaluator.servertest.ts
index b5d97fd4be..eda04540eb 100644
--- a/web/src/__tests__/server/unit/evaluator-v2-test-evaluator.servertest.ts
+++ b/web/src/__tests__/server/unit/evaluator-v2-test-evaluator.servertest.ts
@@ -227,6 +227,7 @@ describe("testEvaluator", () => {
metadata: { customerTier: "pro" },
tool_calls: [],
tool_call_names: [],
+ experiment_id: "experiment-1",
experiment_item_expected_output: "It arrives tomorrow.",
experiment_item_metadata: { source: "test" },
} as never);
@@ -271,6 +272,7 @@ describe("testEvaluator", () => {
},
{ var: "experimentItemMetadata", value: { source: "test" } },
],
+ hasExperimentContext: true,
evaluationContext: {
evaluatorId: "evaluator-1",
evaluatorExecutionIsTest: true,
diff --git a/web/src/features/evals/v2/server/evaluators/testEvaluator.ts b/web/src/features/evals/v2/server/evaluators/testEvaluator.ts
index 7a8f87cd96..1e22eb5d6d 100644
--- a/web/src/features/evals/v2/server/evaluators/testEvaluator.ts
+++ b/web/src/features/evals/v2/server/evaluators/testEvaluator.ts
@@ -86,6 +86,7 @@ export async function testEvaluator(params: {
const result = await runEvaluatorTest({
...params,
variables,
+ hasExperimentContext: Boolean(observation.experiment_id),
...executionData,
});
@@ -98,6 +99,7 @@ async function runEvaluatorTest(params: {
evaluatorId: string;
definition: NormalizedEvaluatorDefinition;
variables: ExtractedVariable[];
+ hasExperimentContext: boolean;
executionMetadata: Record<string, string>;
evaluationContext: ReturnType<
typeof buildEvalExecutionData
@@ -326,6 +328,7 @@ async function testCodeEvaluator(params: {
evaluatorId: string;
definition: Extract<NormalizedEvaluatorDefinition, { type: "CODE" }>;
variables: ExtractedVariable[];
+ hasExperimentContext: boolean;
executionMetadata: Record<string, string>;
evaluationContext: ReturnType<
typeof buildEvalExecutionData
@@ -349,6 +352,7 @@ async function testCodeEvaluator(params: {
evaluator: { id: params.evaluatorId },
version: params.definition,
extractedVariables: params.variables,
+ hasExperimentContext: params.hasExperimentContext,
traceName: "Test evaluator",
metadata: params.executionMetadata,
evaluationContext: params.evaluationContext,