🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(evals): pass experiment context to v2 code evaluator test runs (#19167)
Tobias Wochinger61937211f01a
diff --git a/web/src/__tests__/server/unit/evaluator-v2-test-evaluator.servertest.ts b/web/src/__tests__/server/unit/evaluator-v2-test-evaluator.servertest.tsindex b5d97fd4be..eda04540eb 100644--- a/web/src/__tests__/server/unit/evaluator-v2-test-evaluator.servertest.ts+++ b/web/src/__tests__/server/unit/evaluator-v2-test-evaluator.servertest.ts@@ -227,6 +227,7 @@ describe("testEvaluator", () => {metadata: { customerTier: "pro" },tool_calls: [],tool_call_names: [],+ experiment_id: "experiment-1",experiment_item_expected_output: "It arrives tomorrow.",experiment_item_metadata: { source: "test" },} as never);@@ -271,6 +272,7 @@ describe("testEvaluator", () => {},{ var: "experimentItemMetadata", value: { source: "test" } },],+ hasExperimentContext: true,evaluationContext: {evaluatorId: "evaluator-1",evaluatorExecutionIsTest: true,diff --git a/web/src/features/evals/v2/server/evaluators/testEvaluator.ts b/web/src/features/evals/v2/server/evaluators/testEvaluator.tsindex 7a8f87cd96..1e22eb5d6d 100644--- a/web/src/features/evals/v2/server/evaluators/testEvaluator.ts+++ b/web/src/features/evals/v2/server/evaluators/testEvaluator.ts@@ -86,6 +86,7 @@ export async function testEvaluator(params: {const result = await runEvaluatorTest({...params,variables,+ hasExperimentContext: Boolean(observation.experiment_id),...executionData,});@@ -98,6 +99,7 @@ async function runEvaluatorTest(params: {evaluatorId: string;definition: NormalizedEvaluatorDefinition;variables: ExtractedVariable[];+ hasExperimentContext: boolean;executionMetadata: Record<string, string>;evaluationContext: ReturnType<typeof buildEvalExecutionData@@ -326,6 +328,7 @@ async function testCodeEvaluator(params: {evaluatorId: string;definition: Extract<NormalizedEvaluatorDefinition, { type: "CODE" }>;variables: ExtractedVariable[];+ hasExperimentContext: boolean;executionMetadata: Record<string, string>;evaluationContext: ReturnType<typeof buildEvalExecutionData@@ -349,6 +352,7 @@ async function testCodeEvaluator(params: {evaluator: { id: params.evaluatorId },version: params.definition,extractedVariables: params.variables,+ hasExperimentContext: params.hasExperimentContext,traceName: "Test evaluator",metadata: params.executionMetadata,evaluationContext: params.evaluationContext,