🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(evals): add dedicated evaluator execution columns (#16486)
Tobias Wochinger198e9494fe10
diff --git a/packages/shared/clickhouse/migrations/clustered/0047_add_eval_execution_columns.down.sql b/packages/shared/clickhouse/migrations/clustered/0047_add_eval_execution_columns.down.sqlnew file mode 100644index 0000000000..490efbde32--- /dev/null+++ b/packages/shared/clickhouse/migrations/clustered/0047_add_eval_execution_columns.down.sql@@ -0,0 +1,88 @@+ALTER TABLE events_core_mv ON CLUSTER default MODIFY QUERY+SELECT+ project_id,+ trace_id,+ span_id,+ parent_span_id,+ start_time,+ end_time,+ name,+ type,+ environment,+ version,+ release,+ trace_name,+ user_id,+ session_id,+ tags,+ level,+ status_message,+ completion_start_time,+ is_app_root,+ bookmarked,+ public,+ prompt_id,+ prompt_name,+ prompt_version,+ model_id,+ provided_model_name,+ model_parameters,+ provided_usage_details,+ usage_details,+ provided_cost_details,+ cost_details,+ usage_pricing_tier_id,+ usage_pricing_tier_name,+ tool_definitions,+ tool_calls,+ tool_call_names,+ leftUTF8(input, 200) as input,+ leftUTF8(output, 200) as output,+ metadata_names,+ arrayMap(v -> leftUTF8(v, 200), metadata_values) as metadata_values,+ experiment_id,+ experiment_name,+ experiment_metadata_names,+ experiment_metadata_values,+ experiment_description,+ experiment_dataset_id,+ experiment_item_id,+ experiment_item_version,+ experiment_item_expected_output,+ experiment_item_metadata_names,+ experiment_item_metadata_values,+ experiment_item_root_span_id,+ source,+ service_name,+ service_version,+ scope_name,+ scope_version,+ telemetry_sdk_language,+ telemetry_sdk_name,+ telemetry_sdk_version,+ blob_storage_file_path,+ event_bytes,+ created_at,+ updated_at,+ event_ts,+ is_deleted,+ ingestion_api_key,+ ingestion_sdk_name,+ ingestion_sdk_version+FROM events_full+SETTINGS enable_full_text_index = 1;++ALTER TABLE events_full ON CLUSTER default DROP COLUMN IF EXISTS evaluator_execution_is_test SETTINGS enable_full_text_index = 1, alter_sync = 2;+ALTER TABLE events_full ON CLUSTER default DROP COLUMN IF EXISTS evaluator_id SETTINGS enable_full_text_index = 1, alter_sync = 2;+ALTER TABLE events_full ON CLUSTER default DROP COLUMN IF EXISTS evaluation_rule_id SETTINGS enable_full_text_index = 1, alter_sync = 2;++ALTER TABLE events_core ON CLUSTER default DROP INDEX IF EXISTS idx_evaluator_id SETTINGS enable_full_text_index = 1, alter_sync = 2;+ALTER TABLE events_core ON CLUSTER default DROP INDEX IF EXISTS idx_evaluation_rule_id SETTINGS enable_full_text_index = 1, alter_sync = 2;+ALTER TABLE events_core ON CLUSTER default DROP COLUMN IF EXISTS evaluator_execution_is_test SETTINGS enable_full_text_index = 1, alter_sync = 2;+ALTER TABLE events_core ON CLUSTER default DROP COLUMN IF EXISTS evaluator_id SETTINGS enable_full_text_index = 1, alter_sync = 2;+ALTER TABLE events_core ON CLUSTER default DROP COLUMN IF EXISTS evaluation_rule_id SETTINGS enable_full_text_index = 1, alter_sync = 2;++ALTER TABLE scores ON CLUSTER default DROP INDEX IF EXISTS idx_evaluator_id SETTINGS alter_sync = 2;+ALTER TABLE scores ON CLUSTER default DROP INDEX IF EXISTS idx_evaluation_rule_id SETTINGS alter_sync = 2;+ALTER TABLE scores ON CLUSTER default DROP COLUMN IF EXISTS evaluator_id SETTINGS alter_sync = 2;+ALTER TABLE scores ON CLUSTER default DROP COLUMN IF EXISTS evaluation_rule_id SETTINGS alter_sync = 2;diff --git a/packages/shared/clickhouse/migrations/clustered/0047_add_eval_execution_columns.up.sql b/packages/shared/clickhouse/migrations/clustered/0047_add_eval_execution_columns.up.sqlnew file mode 100644index 0000000000..87504a51b4--- /dev/null+++ b/packages/shared/clickhouse/migrations/clustered/0047_add_eval_execution_columns.up.sql@@ -0,0 +1,113 @@+-- Existing parts resolve these columns through DEFAULT expressions. Skip indexes are added without materializing historical parts to keep this migration metadata-only.+ALTER TABLE events_full ON CLUSTER default+ ADD COLUMN IF NOT EXISTS evaluator_id String DEFAULT arrayElement(metadata_values, indexOf(metadata_names, 'evaluator_id')) AFTER metadata_values+ SETTINGS enable_full_text_index = 1, alter_sync = 2;+ALTER TABLE events_full ON CLUSTER default+ ADD COLUMN IF NOT EXISTS evaluation_rule_id String DEFAULT if(notEmpty(arrayElement(metadata_values, indexOf(metadata_names, 'evaluation_rule_id'))), arrayElement(metadata_values, indexOf(metadata_names, 'evaluation_rule_id')), arrayElement(metadata_values, indexOf(metadata_names, 'job_configuration_id'))) AFTER evaluator_id+ SETTINGS enable_full_text_index = 1, alter_sync = 2;+ALTER TABLE events_full ON CLUSTER default+ ADD COLUMN IF NOT EXISTS evaluator_execution_is_test Bool DEFAULT has(metadata_names, 'evaluator_test') AND arrayElement(metadata_values, indexOf(metadata_names, 'evaluator_test')) = 'true' AFTER evaluation_rule_id+ SETTINGS enable_full_text_index = 1, alter_sync = 2;+ALTER TABLE events_core ON CLUSTER default+ ADD COLUMN IF NOT EXISTS evaluator_id String DEFAULT arrayElement(metadata_values, indexOf(metadata_names, 'evaluator_id')) AFTER metadata_values+ SETTINGS enable_full_text_index = 1, alter_sync = 2;+ALTER TABLE events_core ON CLUSTER default+ ADD COLUMN IF NOT EXISTS evaluation_rule_id String DEFAULT if(notEmpty(arrayElement(metadata_values, indexOf(metadata_names, 'evaluation_rule_id'))), arrayElement(metadata_values, indexOf(metadata_names, 'evaluation_rule_id')), arrayElement(metadata_values, indexOf(metadata_names, 'job_configuration_id'))) AFTER evaluator_id+ SETTINGS enable_full_text_index = 1, alter_sync = 2;+ALTER TABLE events_core ON CLUSTER default+ ADD COLUMN IF NOT EXISTS evaluator_execution_is_test Bool DEFAULT has(metadata_names, 'evaluator_test') AND arrayElement(metadata_values, indexOf(metadata_names, 'evaluator_test')) = 'true' AFTER evaluation_rule_id+ SETTINGS enable_full_text_index = 1, alter_sync = 2;+ALTER TABLE events_core ON CLUSTER default+ ADD INDEX IF NOT EXISTS idx_evaluator_id evaluator_id TYPE bloom_filter(0.01) GRANULARITY 1+ SETTINGS enable_full_text_index = 1, alter_sync = 2;+ALTER TABLE events_core ON CLUSTER default+ ADD INDEX IF NOT EXISTS idx_evaluation_rule_id evaluation_rule_id TYPE bloom_filter(0.01) GRANULARITY 1+ SETTINGS enable_full_text_index = 1, alter_sync = 2;+ALTER TABLE scores ON CLUSTER default+ ADD COLUMN IF NOT EXISTS evaluator_id String DEFAULT metadata['evaluator_id'] AFTER metadata+ SETTINGS alter_sync = 2;+ALTER TABLE scores ON CLUSTER default+ ADD COLUMN IF NOT EXISTS evaluation_rule_id String DEFAULT if(notEmpty(metadata['evaluation_rule_id']), metadata['evaluation_rule_id'], metadata['job_configuration_id']) AFTER evaluator_id+ SETTINGS alter_sync = 2;+ALTER TABLE scores ON CLUSTER default+ ADD INDEX IF NOT EXISTS idx_evaluator_id evaluator_id TYPE bloom_filter(0.001) GRANULARITY 1+ SETTINGS alter_sync = 2;+ALTER TABLE scores ON CLUSTER default+ ADD INDEX IF NOT EXISTS idx_evaluation_rule_id evaluation_rule_id TYPE bloom_filter(0.001) GRANULARITY 1+ SETTINGS alter_sync = 2;+ALTER TABLE events_core_mv ON CLUSTER default MODIFY QUERY+SELECT+ project_id,+ trace_id,+ span_id,+ parent_span_id,+ start_time,+ end_time,+ name,+ type,+ environment,+ version,+ release,+ trace_name,+ user_id,+ session_id,+ tags,+ level,+ status_message,+ completion_start_time,+ is_app_root,+ bookmarked,+ public,+ prompt_id,+ prompt_name,+ prompt_version,+ model_id,+ provided_model_name,+ model_parameters,+ provided_usage_details,+ usage_details,+ provided_cost_details,+ cost_details,+ usage_pricing_tier_id,+ usage_pricing_tier_name,+ tool_definitions,+ tool_calls,+ tool_call_names,+ leftUTF8(input, 200) as input,+ leftUTF8(output, 200) as output,+ metadata_names,+ arrayMap(v -> leftUTF8(v, 200), metadata_values) as metadata_values,+ evaluator_id,+ evaluation_rule_id,+ evaluator_execution_is_test,+ experiment_id,+ experiment_name,+ experiment_metadata_names,+ experiment_metadata_values,+ experiment_description,+ experiment_dataset_id,+ experiment_item_id,+ experiment_item_version,+ experiment_item_expected_output,+ experiment_item_metadata_names,+ experiment_item_metadata_values,+ experiment_item_root_span_id,+ source,+ service_name,+ service_version,+ scope_name,+ scope_version,+ telemetry_sdk_language,+ telemetry_sdk_name,+ telemetry_sdk_version,+ blob_storage_file_path,+ event_bytes,+ created_at,+ updated_at,+ event_ts,+ is_deleted,+ ingestion_api_key,+ ingestion_sdk_name,+ ingestion_sdk_version+FROM events_full+SETTINGS enable_full_text_index = 1;diff --git a/packages/shared/clickhouse/migrations/unclustered/0047_add_eval_execution_columns.down.sql b/packages/shared/clickhouse/migrations/unclustered/0047_add_eval_execution_columns.down.sqlnew file mode 100644index 0000000000..6f1ed617d1--- /dev/null+++ b/packages/shared/clickhouse/migrations/unclustered/0047_add_eval_execution_columns.down.sql@@ -0,0 +1,88 @@+ALTER TABLE events_core_mv MODIFY QUERY+SELECT+ project_id,+ trace_id,+ span_id,+ parent_span_id,+ start_time,+ end_time,+ name,+ type,+ environment,+ version,+ release,+ trace_name,+ user_id,+ session_id,+ tags,+ level,+ status_message,+ completion_start_time,+ is_app_root,+ bookmarked,+ public,+ prompt_id,+ prompt_name,+ prompt_version,+ model_id,+ provided_model_name,+ model_parameters,+ provided_usage_details,+ usage_details,+ provided_cost_details,+ cost_details,+ usage_pricing_tier_id,+ usage_pricing_tier_name,+ tool_definitions,+ tool_calls,+ tool_call_names,+ leftUTF8(input, 200) as input,+ leftUTF8(output, 200) as output,+ metadata_names,+ arrayMap(v -> leftUTF8(v, 200), metadata_values) as metadata_values,+ experiment_id,+ experiment_name,+ experiment_metadata_names,+ experiment_metadata_values,+ experiment_description,+ experiment_dataset_id,+ experiment_item_id,+ experiment_item_version,+ experiment_item_expected_output,+ experiment_item_metadata_names,+ experiment_item_metadata_values,+ experiment_item_root_span_id,+ source,+ service_name,+ service_version,+ scope_name,+ scope_version,+ telemetry_sdk_language,+ telemetry_sdk_name,+ telemetry_sdk_version,+ blob_storage_file_path,+ event_bytes,+ created_at,+ updated_at,+ event_ts,+ is_deleted,+ ingestion_api_key,+ ingestion_sdk_name,+ ingestion_sdk_version+FROM events_full+SETTINGS enable_full_text_index = 1;++ALTER TABLE events_full DROP COLUMN IF EXISTS evaluator_execution_is_test SETTINGS enable_full_text_index = 1;+ALTER TABLE events_full DROP COLUMN IF EXISTS evaluator_id SETTINGS enable_full_text_index = 1;+ALTER TABLE events_full DROP COLUMN IF EXISTS evaluation_rule_id SETTINGS enable_full_text_index = 1;++ALTER TABLE events_core DROP INDEX IF EXISTS idx_evaluator_id SETTINGS enable_full_text_index = 1;+ALTER TABLE events_core DROP INDEX IF EXISTS idx_evaluation_rule_id SETTINGS enable_full_text_index = 1;+ALTER TABLE events_core DROP COLUMN IF EXISTS evaluator_execution_is_test SETTINGS enable_full_text_index = 1;+ALTER TABLE events_core DROP COLUMN IF EXISTS evaluator_id SETTINGS enable_full_text_index = 1;+ALTER TABLE events_core DROP COLUMN IF EXISTS evaluation_rule_id SETTINGS enable_full_text_index = 1;++ALTER TABLE scores DROP INDEX IF EXISTS idx_evaluator_id;+ALTER TABLE scores DROP INDEX IF EXISTS idx_evaluation_rule_id;+ALTER TABLE scores DROP COLUMN IF EXISTS evaluator_id;+ALTER TABLE scores DROP COLUMN IF EXISTS evaluation_rule_id;diff --git a/packages/shared/clickhouse/migrations/unclustered/0047_add_eval_execution_columns.up.sql b/packages/shared/clickhouse/migrations/unclustered/0047_add_eval_execution_columns.up.sqlnew file mode 100644index 0000000000..6a979c92f4--- /dev/null+++ b/packages/shared/clickhouse/migrations/unclustered/0047_add_eval_execution_columns.up.sql@@ -0,0 +1,109 @@+-- Existing parts resolve these columns through DEFAULT expressions. Skip indexes are added without materializing historical parts to keep this migration metadata-only.+ALTER TABLE events_full+ ADD COLUMN IF NOT EXISTS evaluator_id String DEFAULT arrayElement(metadata_values, indexOf(metadata_names, 'evaluator_id')) AFTER metadata_values+ SETTINGS enable_full_text_index = 1;+ALTER TABLE events_full+ ADD COLUMN IF NOT EXISTS evaluation_rule_id String DEFAULT if(notEmpty(arrayElement(metadata_values, indexOf(metadata_names, 'evaluation_rule_id'))), arrayElement(metadata_values, indexOf(metadata_names, 'evaluation_rule_id')), arrayElement(metadata_values, indexOf(metadata_names, 'job_configuration_id'))) AFTER evaluator_id+ SETTINGS enable_full_text_index = 1;+ALTER TABLE events_full+ ADD COLUMN IF NOT EXISTS evaluator_execution_is_test Bool DEFAULT has(metadata_names, 'evaluator_test') AND arrayElement(metadata_values, indexOf(metadata_names, 'evaluator_test')) = 'true' AFTER evaluation_rule_id+ SETTINGS enable_full_text_index = 1;+ALTER TABLE events_core+ ADD COLUMN IF NOT EXISTS evaluator_id String DEFAULT arrayElement(metadata_values, indexOf(metadata_names, 'evaluator_id')) AFTER metadata_values+ SETTINGS enable_full_text_index = 1;+ALTER TABLE events_core+ ADD COLUMN IF NOT EXISTS evaluation_rule_id String DEFAULT if(notEmpty(arrayElement(metadata_values, indexOf(metadata_names, 'evaluation_rule_id'))), arrayElement(metadata_values, indexOf(metadata_names, 'evaluation_rule_id')), arrayElement(metadata_values, indexOf(metadata_names, 'job_configuration_id'))) AFTER evaluator_id+ SETTINGS enable_full_text_index = 1;+ALTER TABLE events_core+ ADD COLUMN IF NOT EXISTS evaluator_execution_is_test Bool DEFAULT has(metadata_names, 'evaluator_test') AND arrayElement(metadata_values, indexOf(metadata_names, 'evaluator_test')) = 'true' AFTER evaluation_rule_id+ SETTINGS enable_full_text_index = 1;+ALTER TABLE events_core+ ADD INDEX IF NOT EXISTS idx_evaluator_id evaluator_id TYPE bloom_filter(0.01) GRANULARITY 1+ SETTINGS enable_full_text_index = 1;+ALTER TABLE events_core+ ADD INDEX IF NOT EXISTS idx_evaluation_rule_id evaluation_rule_id TYPE bloom_filter(0.01) GRANULARITY 1+ SETTINGS enable_full_text_index = 1;+ALTER TABLE scores+ ADD COLUMN IF NOT EXISTS evaluator_id String DEFAULT metadata['evaluator_id'] AFTER metadata;+ALTER TABLE scores+ ADD COLUMN IF NOT EXISTS evaluation_rule_id String DEFAULT if(notEmpty(metadata['evaluation_rule_id']), metadata['evaluation_rule_id'], metadata['job_configuration_id']) AFTER evaluator_id;+ALTER TABLE scores+ ADD INDEX IF NOT EXISTS idx_evaluator_id evaluator_id TYPE bloom_filter(0.001) GRANULARITY 1;+ALTER TABLE scores+ ADD INDEX IF NOT EXISTS idx_evaluation_rule_id evaluation_rule_id TYPE bloom_filter(0.001) GRANULARITY 1;+ALTER TABLE events_core_mv MODIFY QUERY+SELECT+ project_id,+ trace_id,+ span_id,+ parent_span_id,+ start_time,+ end_time,+ name,+ type,+ environment,+ version,+ release,+ trace_name,+ user_id,+ session_id,+ tags,+ level,+ status_message,+ completion_start_time,+ is_app_root,+ bookmarked,+ public,+ prompt_id,+ prompt_name,+ prompt_version,+ model_id,+ provided_model_name,+ model_parameters,+ provided_usage_details,+ usage_details,+ provided_cost_details,+ cost_details,+ usage_pricing_tier_id,+ usage_pricing_tier_name,+ tool_definitions,+ tool_calls,+ tool_call_names,+ leftUTF8(input, 200) as input,+ leftUTF8(output, 200) as output,+ metadata_names,+ arrayMap(v -> leftUTF8(v, 200), metadata_values) as metadata_values,+ evaluator_id,+ evaluation_rule_id,+ evaluator_execution_is_test,+ experiment_id,+ experiment_name,+ experiment_metadata_names,+ experiment_metadata_values,+ experiment_description,+ experiment_dataset_id,+ experiment_item_id,+ experiment_item_version,+ experiment_item_expected_output,+ experiment_item_metadata_names,+ experiment_item_metadata_values,+ experiment_item_root_span_id,+ source,+ service_name,+ service_version,+ scope_name,+ scope_version,+ telemetry_sdk_language,+ telemetry_sdk_name,+ telemetry_sdk_version,+ blob_storage_file_path,+ event_bytes,+ created_at,+ updated_at,+ event_ts,+ is_deleted,+ ingestion_api_key,+ ingestion_sdk_name,+ ingestion_sdk_version+FROM events_full+SETTINGS enable_full_text_index = 1;diff --git a/packages/shared/src/eventsTable.ts b/packages/shared/src/eventsTable.tsindex bd3c2e936c..931fefd8a7 100644--- a/packages/shared/src/eventsTable.ts+++ b/packages/shared/src/eventsTable.ts@@ -1,5 +1,4 @@import { type ColumnDefinition } from "./tableDefinitions";-import { EvalExecutionMetadataKey } from "./features/evals/evalExecutionMetadata";export const eventsTableHasParentObservationSql = "e.parent_span_id != ''";export const eventsTableIsRootObservationSqlForAlias = (alias: string) => {@@ -372,14 +371,14 @@ const eventsTableColsDefinition = [name: "Evaluator ID",id: "evaluatorId",type: "stringOptions",- internal: `arrayElement(e.metadata_values, indexOf(e.metadata_names, '${EvalExecutionMetadataKey.EVALUATOR_ID}'))`,+ internal: "e.evaluator_id",options: [],},{name: "Rule ID",id: "ruleId",type: "stringOptions",- internal: `if(notEmpty(arrayElement(e.metadata_values, indexOf(e.metadata_names, '${EvalExecutionMetadataKey.EVALUATION_RULE_ID}'))), arrayElement(e.metadata_values, indexOf(e.metadata_names, '${EvalExecutionMetadataKey.EVALUATION_RULE_ID}')), arrayElement(e.metadata_values, indexOf(e.metadata_names, '${EvalExecutionMetadataKey.JOB_CONFIGURATION_ID}')))`,+ internal: "e.evaluation_rule_id",options: [],},{diff --git a/packages/shared/src/features/evals/evalExecutionMetadata.ts b/packages/shared/src/features/evals/evalExecutionMetadata.tsindex f6c7823ce6..2d350c1fbb 100644--- a/packages/shared/src/features/evals/evalExecutionMetadata.ts+++ b/packages/shared/src/features/evals/evalExecutionMetadata.ts@@ -13,3 +13,9 @@ export const EvalExecutionMetadataKey = {export type EvalExecutionMetadataKey =(typeof EvalExecutionMetadataKey)[keyof typeof EvalExecutionMetadataKey];++export type EvalExecutionContext = {+ evaluatorId?: string;+ evaluationRuleId?: string;+ evaluatorExecutionIsTest: boolean;+};diff --git a/packages/shared/src/features/query/dataModel.ts b/packages/shared/src/features/query/dataModel.tsindex e007b7e833..2a20ff147d 100644--- a/packages/shared/src/features/query/dataModel.ts+++ b/packages/shared/src/features/query/dataModel.ts@@ -13,7 +13,6 @@ import {eventsTableTraceNameSqlForAlias,} from "../../eventsTable";import { LISTABLE_SCORE_TYPES } from "../../domain/scores";-import { EvalExecutionMetadataKey } from "../evals/evalExecutionMetadata";// The data model defines all available dimensions, measures, and the timeDimension for a given view.// Make sure to update web/src/features/dashboard/lib/dashboardUiTableToViewMapping.ts if you make changes@@ -888,11 +887,19 @@ const createScoreSpecificDimensions = (description: "Name of the score (e.g., accuracy, toxicity).",},evaluatorId: {- sql: `coalesce(nullIf(${tableAlias}.metadata['${EvalExecutionMetadataKey.EVALUATOR_ID}'], ''), ${tableAlias}.metadata['${EvalExecutionMetadataKey.JOB_CONFIGURATION_ID}'])`,+ sql: `${tableAlias}.evaluator_id`,alias: "evaluatorId",type: "string",+ description: "Identifier of the evaluator that produced the score.",+ highCardinality: true,+ uiHidden: true,+ },+ ruleId: {+ sql: `${tableAlias}.evaluation_rule_id`,+ alias: "ruleId",+ type: "string",description:- "Identifier of the evaluator, falling back to its legacy evaluation rule identifier.",+ "Identifier of the evaluation rule, including legacy job configurations.",highCardinality: true,uiHidden: true,},diff --git a/packages/shared/src/features/query/server/queryBuilder.filterValidation.test.ts b/packages/shared/src/features/query/server/queryBuilder.filterValidation.test.tsindex 329baa129a..afadf42dfd 100644--- a/packages/shared/src/features/query/server/queryBuilder.filterValidation.test.ts+++ b/packages/shared/src/features/query/server/queryBuilder.filterValidation.test.ts@@ -290,9 +290,26 @@ describe("queryBuilder filter type validation", () => {version,);- expect(query).toContain(- "coalesce(nullIf(scores_numeric.metadata['evaluator_id'], ''), scores_numeric.metadata['job_configuration_id'])",+ expect(query).toContain("scores_numeric.evaluator_id");+ expect(query).toContain("IN ({stringOptionsFilter");+ },+ );++ it.each(["v1", "v2"] as const)(+ "lowers rule score filters in the %s scores view",+ async (version) => {+ const { query } = await buildQueryWithFilter(+ {+ column: "ruleId",+ operator: "any of",+ value: ["rule-1"],+ type: "stringOptions",+ },+ { view: "scores-numeric" },+ version,);++ expect(query).toContain("scores_numeric.evaluation_rule_id");expect(query).toContain("IN ({stringOptionsFilter");},);diff --git a/packages/shared/src/server/evals/codeEvalExecution.ts b/packages/shared/src/server/evals/codeEvalExecution.tsindex 233016f940..3ab20b8f71 100644--- a/packages/shared/src/server/evals/codeEvalExecution.ts+++ b/packages/shared/src/server/evals/codeEvalExecution.ts@@ -1,4 +1,5 @@import { stringifyValue } from "../../utils/stringChecks";+import type { EvalExecutionContext } from "../../features/evals/evalExecutionMetadata";import {INTERNAL_TRACE_EVENT_SOURCE,type InternalTraceEventInput,@@ -221,6 +222,7 @@ export async function runCodeBasedEvaluationDispatch(params: {hasExperimentContext?: boolean;traceName: string;metadata: Record<string, unknown>;+ evaluationContext?: EvalExecutionContext;writeTrace?: InternalTraceWriter;}): Promise<CodeBasedEvaluationDispatchResult> {const payload = buildCodeEvalPayload({@@ -253,6 +255,7 @@ export async function runCodeBasedEvaluationDispatch(params: {payload,output: dispatchResult,metadata: params.metadata,+ evaluationContext: params.evaluationContext,sourceCode: params.version.sourceCode,}),});@@ -300,6 +303,7 @@ export async function runCodeBasedEvaluationDispatch(params: {: {}),error_retryable: errorDetails.retryable,},+ evaluationContext: params.evaluationContext,sourceCode: params.version.sourceCode,level: "ERROR",statusMessage: `Code eval execution failed: ${visibleError.message}`,@@ -323,6 +327,7 @@ function buildCodeEvalTraceInput(params: {payload: CodeEvalPayload;output: unknown;metadata: Record<string, unknown>;+ evaluationContext?: EvalExecutionContext;sourceCode: string;level?: string;statusMessage?: string;@@ -345,6 +350,7 @@ function buildCodeEvalTraceInput(params: {...params.metadata,code_eval_source_code: params.sourceCode,},+ evaluationContext: params.evaluationContext,source: INTERNAL_TRACE_EVENT_SOURCE,};diff --git a/packages/shared/src/server/evals/evalExecutionMetadata.test.ts b/packages/shared/src/server/evals/evalExecutionMetadata.test.tsindex 4b4db392fc..f00506a838 100644--- a/packages/shared/src/server/evals/evalExecutionMetadata.test.ts+++ b/packages/shared/src/server/evals/evalExecutionMetadata.test.ts@@ -1,26 +1,32 @@import { describe, expect, it } from "vitest";-import { buildEvalExecutionMetadata } from "./evalExecutionMetadata";+import { buildEvalExecutionData } from "./evalExecutionMetadata";-describe("buildEvalExecutionMetadata", () => {+describe("buildEvalExecutionData", () => {it("identifies an evaluator test", () => {expect(- buildEvalExecutionMetadata({+ buildEvalExecutionData({type: "TEST",evaluatorId: "evaluator-1",targetTraceId: "trace-1",targetObservationId: "observation-1",}),).toEqual({- evaluator_id: "evaluator-1",- evaluator_test: "true",- target_trace_id: "trace-1",- target_observation_id: "observation-1",+ executionMetadata: {+ evaluator_id: "evaluator-1",+ evaluator_test: "true",+ target_trace_id: "trace-1",+ target_observation_id: "observation-1",+ },+ evaluationContext: {+ evaluatorId: "evaluator-1",+ evaluatorExecutionIsTest: true,+ },});});it("identifies a rule-driven evaluator execution", () => {expect(- buildEvalExecutionMetadata({+ buildEvalExecutionData({type: "JOB",jobExecutionId: "job-1",jobConfigurationId: "rule-1",@@ -33,20 +39,27 @@ describe("buildEvalExecutionMetadata", () => {targetDatasetItemId: null,}),).toEqual({- job_execution_id: "job-1",- evaluation_rule_id: "rule-1",- job_configuration_id: "rule-1",- evaluation_rule_assignment_id: "assignment-1",- evaluator_id: "evaluator-1",- evaluator_version_id: "version-2",- target_trace_id: "trace-1",- target_observation_id: "observation-1",+ executionMetadata: {+ job_execution_id: "job-1",+ job_configuration_id: "rule-1",+ evaluation_rule_id: "rule-1",+ evaluation_rule_assignment_id: "assignment-1",+ evaluator_version_id: "version-2",+ evaluator_id: "evaluator-1",+ target_trace_id: "trace-1",+ target_observation_id: "observation-1",+ },+ evaluationContext: {+ evaluatorId: "evaluator-1",+ evaluationRuleId: "rule-1",+ evaluatorExecutionIsTest: false,+ },});});it("identifies a manual v2 evaluator execution without a rule", () => {expect(- buildEvalExecutionMetadata({+ buildEvalExecutionData({type: "JOB",jobExecutionId: "job-1",jobConfigurationId: "evaluator-1",@@ -57,10 +70,17 @@ describe("buildEvalExecutionMetadata", () => {targetDatasetItemId: null,}),).toEqual({- job_execution_id: "job-1",- job_configuration_id: "evaluator-1",- evaluator_id: "evaluator-1",- evaluator_version_id: "version-2",+ executionMetadata: {+ job_execution_id: "job-1",+ job_configuration_id: "evaluator-1",+ evaluator_version_id: "version-2",+ evaluator_id: "evaluator-1",+ },+ evaluationContext: {+ evaluatorId: "evaluator-1",+ evaluationRuleId: "evaluator-1",+ evaluatorExecutionIsTest: false,+ },});});});diff --git a/packages/shared/src/server/evals/evalExecutionMetadata.ts b/packages/shared/src/server/evals/evalExecutionMetadata.tsindex 51bf4655a9..7ccc18c685 100644--- a/packages/shared/src/server/evals/evalExecutionMetadata.ts+++ b/packages/shared/src/server/evals/evalExecutionMetadata.ts@@ -1,6 +1,9 @@-import { EvalExecutionMetadataKey } from "../../features/evals/evalExecutionMetadata";+import {+ EvalExecutionMetadataKey,+ type EvalExecutionContext,+} from "../../features/evals/evalExecutionMetadata";-export function buildEvalExecutionMetadata(+export function buildEvalExecutionData(params:| {type: "TEST";@@ -21,7 +24,9 @@ export function buildEvalExecutionMetadata(targetDatasetItemId: string | null;},) {- const metadata =+ // Keep old workers and legacy trace/observation sinks compatible during the+ // v4 rollout. This metadata transport can be removed in v5.+ const executionMetadata =params.type === "TEST"? {[EvalExecutionMetadataKey.EVALUATOR_ID]: params.evaluatorId,@@ -46,9 +51,9 @@ export function buildEvalExecutionMetadata(params.assignmentId,}: {}),- [EvalExecutionMetadataKey.EVALUATOR_ID]: params.evaluatorId,[EvalExecutionMetadataKey.EVALUATOR_VERSION_ID]:params.evaluatorVersionId,+ [EvalExecutionMetadataKey.EVALUATOR_ID]: params.evaluatorId,[EvalExecutionMetadataKey.TARGET_TRACE_ID]: params.targetTraceId,[EvalExecutionMetadataKey.TARGET_OBSERVATION_ID]:params.targetObservationId,@@ -56,7 +61,23 @@ export function buildEvalExecutionMetadata(params.targetDatasetItemId,};- return Object.fromEntries(- Object.entries(metadata).filter(([, value]) => value != null),- ) as Record<string, string>;+ const evaluationContext: EvalExecutionContext =+ params.type === "TEST"+ ? {+ evaluatorId: params.evaluatorId ?? undefined,+ evaluatorExecutionIsTest: true,+ }+ : {+ evaluatorId: params.evaluatorId,+ evaluationRuleId:+ params.evaluationRuleId ?? params.jobConfigurationId,+ evaluatorExecutionIsTest: false,+ };++ return {+ executionMetadata: Object.fromEntries(+ Object.entries(executionMetadata).filter(([, value]) => value != null),+ ) as Record<string, string>,+ evaluationContext,+ };}diff --git a/packages/shared/src/server/llm/ai-sdk/telemetry.ts b/packages/shared/src/server/llm/ai-sdk/telemetry.tsindex b0d6cfcaa1..3b266de579 100644--- a/packages/shared/src/server/llm/ai-sdk/telemetry.ts+++ b/packages/shared/src/server/llm/ai-sdk/telemetry.ts@@ -25,6 +25,7 @@ import { traceException } from "../../instrumentation";import { logger } from "../../logger";import {AI_FEATURE_OTEL_SDK_NAME,+ buildEvaluationAttributes,LangfuseOtelSpanAttributes,} from "../../otel/attributes";import { publishInternalOtelSpans } from "../../otel/internalTraceOtelWriter";@@ -146,6 +147,9 @@ export function createAiSdkTelemetryCapture(params: {const serializedInput =rootInput !== undefined ? stringifyValue(rootInput) : undefined;+ const evaluationAttributes = traceSinkParams.evaluationContext+ ? buildEvaluationAttributes(traceSinkParams.evaluationContext)+ : undefined;const rootSpan: Span = tracer.startSpan(traceSinkParams.traceName,@@ -159,6 +163,7 @@ export function createAiSdkTelemetryCapture(params: {traceSinkParams.userId,}: {}),+ ...(evaluationAttributes ?? {}),...(traceSinkParams.metadata? {[LangfuseOtelSpanAttributes.TRACE_METADATA]: JSON.stringify(@@ -214,6 +219,7 @@ export function createAiSdkTelemetryCapture(params: {// Experiment linkage goes on every span so every materialized event// remains associated with the run item root....(experimentAttributes ?? {}),+ ...(evaluationAttributes ?? {}),...(promptAttributes ?? {}),[LangfuseOtelSpanAttributes.TRACE_NAME]: traceSinkParams.traceName,[LangfuseOtelSpanAttributes.ENVIRONMENT]: traceSinkParams.environment,diff --git a/packages/shared/src/server/llm/ai-sdk/telemetryIntegration.test.ts b/packages/shared/src/server/llm/ai-sdk/telemetryIntegration.test.tsindex 9667cb937e..6c324bc7a6 100644--- a/packages/shared/src/server/llm/ai-sdk/telemetryIntegration.test.ts+++ b/packages/shared/src/server/llm/ai-sdk/telemetryIntegration.test.ts@@ -50,7 +50,18 @@ const traceSinkParams: TraceSinkParams = {traceId: VALID_TRACE_ID,traceName: "Execute evaluator: helpfulness",environment: "langfuse-llm-judge",- metadata: { job_configuration_id: "evaluator-1" },+ metadata: {+ job_execution_id: "job-1",+ evaluator_id: "evaluator-1",+ evaluation_rule_id: "legacy-rule-1",+ job_configuration_id: "legacy-rule-1",+ evaluator_test: "true",+ },+ evaluationContext: {+ evaluatorId: "evaluator-1",+ evaluationRuleId: "legacy-rule-1",+ evaluatorExecutionIsTest: true,+ },};const modelParams: ModelParams = {@@ -278,8 +289,44 @@ describe("AI SDK telemetry integration", () => {model: "gpt-4o",usageDetails: { input: 3, output: 5 },environment: "langfuse-llm-judge",- metadata: { job_configuration_id: "evaluator-1" },+ metadata: {+ job_execution_id: "job-1",+ evaluator_id: "evaluator-1",+ evaluation_rule_id: "legacy-rule-1",+ job_configuration_id: "legacy-rule-1",+ evaluator_test: "true",+ },});+ const generationMetadata = (+ generation.body as {+ metadata: { attributes?: Record<string, unknown> };+ }+ ).metadata;+ expect(generationMetadata.attributes).toMatchObject({+ "gen_ai.operation.name": "chat",+ "langfuse.evaluator.id": "evaluator-1",+ "langfuse.evaluation.rule.id": "legacy-rule-1",+ "langfuse.evaluator.execution.is_test": "true",+ });++ const eventInputs = processor.processToEvent(resourceSpans);+ expect(eventInputs).toHaveLength(2);+ for (const eventInput of eventInputs) {+ expect(eventInput).toMatchObject({+ evaluationContext: {+ evaluatorId: "evaluator-1",+ evaluationRuleId: "legacy-rule-1",+ evaluatorExecutionIsTest: true,+ },+ metadata: {+ job_execution_id: "job-1",+ evaluator_id: "evaluator-1",+ evaluation_rule_id: "legacy-rule-1",+ job_configuration_id: "legacy-rule-1",+ evaluator_test: "true",+ },+ });+ }});it("tags experiment run items so the ingestion pipeline can schedule experiment evals", async () => {diff --git a/packages/shared/src/server/llm/internalTraceEvents.ts b/packages/shared/src/server/llm/internalTraceEvents.tsindex 5b4496c711..325142ab59 100644--- a/packages/shared/src/server/llm/internalTraceEvents.ts+++ b/packages/shared/src/server/llm/internalTraceEvents.ts@@ -5,6 +5,7 @@ import {asString,asStringArray,} from "../../utils/objects";+import type { EvalExecutionContext } from "../../features/evals/evalExecutionMetadata";import { stringifyValue } from "../../utils/stringChecks";import {ObservationTypeDomain,@@ -92,6 +93,7 @@ export type InternalTraceEventInput = {blobStorageFilePath?: string;eventRaw?: string;eventBytes?: number;+ evaluationContext?: EvalExecutionContext;experimentId?: string;experimentName?: string;experimentMetadataNames?: string[];diff --git a/packages/shared/src/server/llm/types.ts b/packages/shared/src/server/llm/types.tsindex caf1e8af41..be393ccc0b 100644--- a/packages/shared/src/server/llm/types.ts+++ b/packages/shared/src/server/llm/types.ts@@ -10,6 +10,7 @@ import type {InternalTraceEventInput,InternalTraceExperimentContext,} from "./internalTraceEvents";+import type { EvalExecutionContext } from "../../features/evals/evalExecutionMetadata";// disable lint as this is exported and used in web/worker@@ -543,6 +544,7 @@ export type TraceSinkParams = {userId?: string;sessionId?: string;metadata?: Record<string, unknown>;+ evaluationContext?: EvalExecutionContext;prompt?: {name: string;version: number;diff --git a/packages/shared/src/server/otel/OtelIngestionProcessor.ts b/packages/shared/src/server/otel/OtelIngestionProcessor.tsindex 5334f756c1..d93dedfbd9 100644--- a/packages/shared/src/server/otel/OtelIngestionProcessor.ts+++ b/packages/shared/src/server/otel/OtelIngestionProcessor.ts@@ -494,6 +494,8 @@ export class OtelIngestionProcessor {spanAttributes,span,);+ const evaluationFields =+ this.extractEvaluationFields(spanAttributes);const spanContext = {spanId,@@ -658,6 +660,9 @@ export class OtelIngestionProcessor {// Experiment fields...experimentFields,+ // Evaluator execution fields+ ...evaluationFields,+// Tool callingtoolDefinitions,toolCalls,@@ -3499,6 +3504,34 @@ export class OtelIngestionProcessor {};}+ private extractEvaluationFields(attributes: Record<string, unknown>) {+ const evaluatorId = attributes[LangfuseOtelSpanAttributes.EVALUATOR_ID];+ const evaluationRuleId =+ attributes[LangfuseOtelSpanAttributes.EVALUATION_RULE_ID];+ const evaluatorExecutionIsTest =+ attributes[LangfuseOtelSpanAttributes.EVALUATOR_EXECUTION_IS_TEST];++ if (+ evaluatorId == null &&+ evaluationRuleId == null &&+ evaluatorExecutionIsTest == null+ ) {+ return { evaluationContext: undefined };+ }++ return {+ evaluationContext: {+ evaluatorId: evaluatorId ? String(evaluatorId) : undefined,+ evaluationRuleId: evaluationRuleId+ ? String(evaluationRuleId)+ : undefined,+ evaluatorExecutionIsTest:+ evaluatorExecutionIsTest === true ||+ evaluatorExecutionIsTest === "true",+ },+ };+ }+/*** The item version is a pointer to a dataset item version (`valid_from` timestamp),* not a free-form label; "v1" or "latest" cannot resolve to one, so we drop them.diff --git a/packages/shared/src/server/otel/attributes.ts b/packages/shared/src/server/otel/attributes.tsindex 7382b6b50f..21358cab9e 100644--- a/packages/shared/src/server/otel/attributes.ts+++ b/packages/shared/src/server/otel/attributes.ts@@ -1,3 +1,5 @@+import type { EvalExecutionContext } from "../../features/evals/evalExecutionMetadata";+export enum LangfuseOtelSpanAttributes {// Langfuse-Trace attributesTRACE_NAME = "langfuse.trace.name",@@ -51,6 +53,32 @@ export enum LangfuseOtelSpanAttributes {EXPERIMENT_ITEM_METADATA = "langfuse.experiment.item.metadata",EXPERIMENT_ITEM_ROOT_OBSERVATION_ID = "langfuse.experiment.item.root_observation_id",EXPERIMENT_ITEM_EXPECTED_OUTPUT = "langfuse.experiment.item.expected_output",++ // Evaluator execution attributes+ EVALUATOR_ID = "langfuse.evaluator.id",+ EVALUATION_RULE_ID = "langfuse.evaluation.rule.id",+ EVALUATOR_EXECUTION_IS_TEST = "langfuse.evaluator.execution.is_test",+}++export function buildEvaluationAttributes(+ evaluationContext: EvalExecutionContext,+) {+ return {+ ...(evaluationContext.evaluatorId+ ? {+ [LangfuseOtelSpanAttributes.EVALUATOR_ID]:+ evaluationContext.evaluatorId,+ }+ : {}),+ ...(evaluationContext.evaluationRuleId+ ? {+ [LangfuseOtelSpanAttributes.EVALUATION_RULE_ID]:+ evaluationContext.evaluationRuleId,+ }+ : {}),+ [LangfuseOtelSpanAttributes.EVALUATOR_EXECUTION_IS_TEST]:+ evaluationContext.evaluatorExecutionIsTest,+ };}/** SDK name for internally published AI-feature product traces. */diff --git a/packages/shared/src/server/otel/internalTraceOtelWriter.test.ts b/packages/shared/src/server/otel/internalTraceOtelWriter.test.tsindex b2cc517090..e49ad56f12 100644--- a/packages/shared/src/server/otel/internalTraceOtelWriter.test.ts+++ b/packages/shared/src/server/otel/internalTraceOtelWriter.test.ts@@ -37,7 +37,18 @@ const codeEvalRootInput: InternalOtelSpanInput = {statusMessage: "Code eval execution failed: boom",input: '{"item":{}}',output: '{"error":"boom"}',- metadata: { dispatcher_name: "test-dispatcher" },+ metadata: {+ dispatcher_name: "test-dispatcher",+ evaluator_id: "evaluator-1",+ evaluation_rule_id: "legacy-rule-1",+ job_configuration_id: "legacy-rule-1",+ evaluator_test: "true",+ },+ evaluationContext: {+ evaluatorId: "evaluator-1",+ evaluationRuleId: "legacy-rule-1",+ evaluatorExecutionIsTest: true,+ },};// Convert the published OTLP payload through the REAL OTel ingestion@@ -92,6 +103,22 @@ describe("writeInternalTraceViaOtelIngestion", () => {});expect(root.metadata).toMatchObject({dispatcher_name: "test-dispatcher",+ evaluator_id: "evaluator-1",+ evaluation_rule_id: "legacy-rule-1",+ job_configuration_id: "legacy-rule-1",+ evaluator_test: "true",+ });+ expect(root).toMatchObject({+ evaluationContext: {+ evaluatorId: "evaluator-1",+ evaluationRuleId: "legacy-rule-1",+ evaluatorExecutionIsTest: true,+ },+ });+ expect(root.metadata.attributes).toMatchObject({+ "langfuse.evaluator.id": "evaluator-1",+ "langfuse.evaluation.rule.id": "legacy-rule-1",+ "langfuse.evaluator.execution.is_test": "true",});});diff --git a/packages/shared/src/server/otel/internalTraceOtelWriter.ts b/packages/shared/src/server/otel/internalTraceOtelWriter.tsindex 325235cc02..7c5509b90b 100644--- a/packages/shared/src/server/otel/internalTraceOtelWriter.ts+++ b/packages/shared/src/server/otel/internalTraceOtelWriter.ts@@ -18,7 +18,10 @@ import {import { ObservationTypeDomain } from "../../domain/observations";import type { InternalTraceEventInput } from "../llm/internalTraceEvents";import { logger } from "../logger";-import { LangfuseOtelSpanAttributes } from "./attributes";+import {+ buildEvaluationAttributes,+ LangfuseOtelSpanAttributes,+} from "./attributes";import {OtelIngestionProcessor,type ResourceSpan,@@ -69,11 +72,15 @@ export function internalTraceEventToOtelAttributes(| "public"| "release"| "traceName"+ | "evaluationContext">,options: { isRoot: boolean },): Attributes {const observationType = observationTypeToOtelAttribute(eventInput.type);const metadataJson = asOtelAttributeString(eventInput.metadata);+ const evaluationAttributes = eventInput.evaluationContext+ ? buildEvaluationAttributes(eventInput.evaluationContext)+ : undefined;const { isRoot } = options;return {@@ -101,6 +108,7 @@ export function internalTraceEventToOtelAttributes(...(metadataJson !== undefined? { [LangfuseOtelSpanAttributes.OBSERVATION_METADATA]: metadataJson }: {}),+ ...(evaluationAttributes ?? {}),...(eventInput.modelName !== undefined? { [LangfuseOtelSpanAttributes.OBSERVATION_MODEL]: eventInput.modelName }: {}),@@ -271,6 +279,7 @@ export type InternalOtelSpanInput = Pick<| "input"| "output"| "metadata"+ | "evaluationContext">;/**diff --git a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.test.ts b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.test.tsindex c95e97a516..7fe98aebe8 100644--- a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.test.ts+++ b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.test.ts@@ -38,6 +38,21 @@ describe("EventsAggregationQueryBuilder", () => {"(e.parent_span_id = '' OR e.is_app_root = true) AND e.name <> ''",);});++ it("promotes evaluator execution fields into the trace aggregation", () => {+ const { query } = new EventsAggregationQueryBuilder({+ projectId: "test-project",+ })+ .selectFieldSet("all")+ .buildWithParams();++ expect(query).toContain(+ "argMaxIf(evaluator_id, event_ts, evaluator_id <> '') AS evaluator_id",+ );+ expect(query).toContain(+ "argMaxIf(evaluation_rule_id, event_ts, evaluation_rule_id <> '') AS evaluation_rule_id",+ );+ });});describe("EventsSessionAggregationQueryBuilder", () => {diff --git a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.tsindex c7c75a955d..94220e145f 100644--- a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts+++ b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts@@ -569,6 +569,12 @@ const EVENTS_AGGREGATION_FIELDS = {tags: "argMaxIf(tags, event_ts, notEmpty(tags)) AS tags",release: "argMaxIf(release, event_ts, release <> '') AS release",+ // Evaluator execution fields are stamped on every internal evaluation span.+ evaluator_id:+ "argMaxIf(evaluator_id, event_ts, evaluator_id <> '') AS evaluator_id",+ evaluation_rule_id:+ "argMaxIf(evaluation_rule_id, event_ts, evaluation_rule_id <> '') AS evaluation_rule_id",+// experiment fieldsexperiment_id: "any(experiment_id) as experiment_id",} as const;diff --git a/packages/shared/src/server/repositories/definitions.ts b/packages/shared/src/server/repositories/definitions.tsindex 552d2c2e5e..819b3ff2fa 100644--- a/packages/shared/src/server/repositories/definitions.ts+++ b/packages/shared/src/server/repositories/definitions.ts@@ -225,6 +225,8 @@ export const scoreRecordBaseSchema = z.object({source: z.string(),comment: z.string().nullish(),metadata: z.record(z.string(), z.string()),+ evaluator_id: z.string().optional(),+ evaluation_rule_id: z.string().optional(),author_user_id: z.string().nullish(),config_id: z.string().nullish(),data_type: z.string(),@@ -517,6 +519,9 @@ export const eventRecordBaseSchema = z.object({// Metadatametadata_names: z.array(z.string()).default([]),metadata_values: z.array(z.string()).default([]),+ evaluator_id: z.string().optional(),+ evaluation_rule_id: z.string().optional(),+ evaluator_execution_is_test: z.boolean().optional(),// Experiment propertiesexperiment_id: z.string().nullish(),diff --git a/packages/shared/src/server/repositories/events.ts b/packages/shared/src/server/repositories/events.tsindex 941321e60a..9763d91f7e 100644--- a/packages/shared/src/server/repositories/events.ts+++ b/packages/shared/src/server/repositories/events.ts@@ -6,10 +6,6 @@ import type {ObservationType,} from "../../domain";import { env } from "../../env";-import {- EvalExecutionMetadataKey,- type EvalExecutionMetadataKey as EvalExecutionMetadataKeyType,-} from "../../features/evals/evalExecutionMetadata";import {InternalServerError,InvalidRequestError,@@ -3404,34 +3400,16 @@ export const getTraceMetadataByIdsFromEvents = async (props: {});};-const eventMetadataValue = (key: EvalExecutionMetadataKeyType) =>- `arrayElement(e.metadata_values, indexOf(e.metadata_names, '${key}'))`;--/**- * Value of the first key that is present, so callers can read a renamed- * metadata key while executions written before the rename still resolve.- */-const eventMetadataValueWithFallback = (- keys: readonly EvalExecutionMetadataKeyType[],-) =>- keys- .slice(0, -1)- .reduceRight(- (fallback, key) =>- `if(notEmpty(${eventMetadataValue(key)}), ${eventMetadataValue(key)}, ${fallback})`,- eventMetadataValue(keys[keys.length - 1]!),- );--const hasAnyEventMetadataKey = (- keys: readonly EvalExecutionMetadataKeyType[],-) => keys.map((key) => `has(e.metadata_names, '${key}')`).join(" OR ");+const evalExecutionIdentifierColumn = (identifier: "evaluator" | "rule") =>+ identifier === "evaluator" ? "e.evaluator_id" : "e.evaluation_rule_id";-const evaluatorTestEventCondition = `(has(e.metadata_names, '${EvalExecutionMetadataKey.EVALUATOR_TEST}') AND ${eventMetadataValue(EvalExecutionMetadataKey.EVALUATOR_TEST)} = 'true') OR startsWith(e.trace_name, 'Test evaluator')`;+const evaluatorTestEventCondition =+ "e.evaluator_execution_is_test OR startsWith(e.trace_name, 'Test evaluator')";-const traceCostsByMetadata = (params: {+const traceCostsByIdentifier = (params: {projectId: string;- metadata: Array<{- keys: readonly EvalExecutionMetadataKeyType[];+ identifiers: Array<{+ identifier: "evaluator" | "rule";alias: string;}>;additionalSelect?: string[];@@ -3441,10 +3419,10 @@ const traceCostsByMetadata = (params: {groupByColumn: "e.trace_id",selectExpression: ["e.trace_id as trace_id",- ...params.metadata.map(- ({ keys, alias }) =>- `anyIf(${eventMetadataValueWithFallback(keys)}, ${hasAnyEventMetadataKey(keys)}) as ${alias}`,- ),+ ...params.identifiers.map(({ identifier, alias }) => {+ const identifierColumn = evalExecutionIdentifierColumn(identifier);+ return `anyIf(${identifierColumn}, notEmpty(${identifierColumn})) as ${alias}`;+ }),"sum(e.total_cost) as trace_total_cost",...(params.additionalSelect ?? []),].join(", "),@@ -3473,16 +3451,13 @@ const getCostMetricsByMetadataIds = async <projectId: string;metadataIds: string[];fields: TFields;- metadataKeys: readonly Exclude<- EvalExecutionMetadataKeyType,- typeof EvalExecutionMetadataKey.EVALUATOR_TEST- >[];+ identifier: "evaluator" | "rule";}) => {if (params.metadataIds.length === 0) return [];- const traceCostsBuilder = traceCostsByMetadata({+ const traceCostsBuilder = traceCostsByIdentifier({projectId: params.projectId,- metadata: [{ keys: params.metadataKeys, alias: "metadata_id" }],+ identifiers: [{ identifier: params.identifier, alias: "metadata_id" }],additionalSelect: [`countIf(${evaluatorTestEventCondition}) as test_event_count`,],@@ -3533,7 +3508,7 @@ export const getAvgCostByEvaluatorIds = async (projectId,metadataIds: evaluatorIds,fields: ["avgCost", "executionCount"],- metadataKeys: [EvalExecutionMetadataKey.EVALUATOR_ID],+ identifier: "evaluator",});return metrics.map(({ metadataId, ...costs }) => ({evaluatorId: metadataId,@@ -3549,13 +3524,7 @@ export const getTotalCostByRule = async (projectId,metadataIds: ruleIds,fields: ["totalCost"],- // Rules replaced job configurations, but executions written before the- // rename only carry `job_configuration_id`, and the v1 evals UI still- // reads costs by job-configuration id.- metadataKeys: [- EvalExecutionMetadataKey.EVALUATION_RULE_ID,- EvalExecutionMetadataKey.JOB_CONFIGURATION_ID,- ],+ identifier: "rule",});return metrics.map(({ metadataId, totalCost }) => ({ruleId: metadataId,@@ -3571,7 +3540,7 @@ export const getTotalCostByEvaluatorIds = async (projectId,metadataIds: evaluatorIds,fields: ["totalCost"],- metadataKeys: [EvalExecutionMetadataKey.EVALUATOR_ID],+ identifier: "evaluator",});return metrics.map(({ metadataId, totalCost }) => ({evaluatorId: metadataId,@@ -3579,7 +3548,8 @@ export const getTotalCostByEvaluatorIds = async (}));};-// Temporary compatibility path: remove in a few weeks in favor of evaluator_id.+// Prompt-experiment executions still use the trace-name fallback until every+// execution is linked to an evaluator id.export const getTotalCostByEvaluatorTraceNames = async (projectId: string,traceNames: string[],@@ -3635,12 +3605,12 @@ export const getLatestEvaluatorRunCost = async (projectId: string,evaluatorId: string,) => {- const builder = traceCostsByMetadata({+ const builder = traceCostsByIdentifier({projectId,- metadata: [+ identifiers: [{- keys: [EvalExecutionMetadataKey.EVALUATOR_ID],- alias: EvalExecutionMetadataKey.EVALUATOR_ID,+ identifier: "evaluator",+ alias: "evaluator_id",},],additionalSelect: [@@ -3648,10 +3618,7 @@ export const getLatestEvaluatorRunCost = async ("countIf(e.type = 'GENERATION') as generation_count",],})- .havingRaw(- `${EvalExecutionMetadataKey.EVALUATOR_ID} = {evaluatorId: String}`,- { evaluatorId },- )+ .havingRaw("evaluator_id = {evaluatorId: String}", { evaluatorId }).havingRaw("generation_count > 0").orderBy("ORDER BY timestamp DESC, trace_id DESC").limit(1);@@ -3718,29 +3685,20 @@ export const getRecentRuleExecutionTraces = async () => {if (ruleIds.length === 0) return [];- const evaluationRuleId = eventMetadataValueWithFallback([- EvalExecutionMetadataKey.EVALUATION_RULE_ID,- EvalExecutionMetadataKey.JOB_CONFIGURATION_ID,- ]);- const hasEvaluationRuleId = hasAnyEventMetadataKey([- EvalExecutionMetadataKey.EVALUATION_RULE_ID,- EvalExecutionMetadataKey.JOB_CONFIGURATION_ID,- ]);const builder = new EventsAggQueryBuilder({projectId,- groupByColumn: `e.trace_id, ${EvalExecutionMetadataKey.EVALUATION_RULE_ID}`,+ groupByColumn: "e.trace_id, e.evaluation_rule_id",selectExpression: ["e.trace_id as id",- `${evaluationRuleId} as ${EvalExecutionMetadataKey.EVALUATION_RULE_ID}`,+ "e.evaluation_rule_id as evaluation_rule_id","multiIf(countIf(e.level = 'ERROR') > 0, 'ERROR', countIf(e.level = 'WARNING') > 0, 'WARNING', 'DEFAULT') as level","min(e.start_time) as timestamp",].join(", "),}).whereRaw("e.start_time > now() - INTERVAL 7 DAY")- .whereRaw(`(${hasEvaluationRuleId})`)- .whereRaw(`${evaluationRuleId} IN ({ruleIds: Array(String)})`, { ruleIds })+ .whereRaw("e.evaluation_rule_id IN ({ruleIds: Array(String)})", { ruleIds }).orderBy("ORDER BY timestamp DESC, id DESC")- .limitByCount(5, EvalExecutionMetadataKey.EVALUATION_RULE_ID);+ .limitByCount(5, "evaluation_rule_id");const { query, params } = builder.buildWithParams();const rows = await queryClickhouse<@@ -3748,7 +3706,7 @@ export const getRecentRuleExecutionTraces = async (id: string;level: string;timestamp: string;- } & Record<typeof EvalExecutionMetadataKey.EVALUATION_RULE_ID, string>+ } & { evaluation_rule_id: string }>({query,params,@@ -3758,7 +3716,7 @@ export const getRecentRuleExecutionTraces = async (return rows.map((row) => ({id: row.id,- ruleId: row[EvalExecutionMetadataKey.EVALUATION_RULE_ID],+ ruleId: row.evaluation_rule_id,level: row.level,timestamp: parseClickhouseUTCDateTimeFormat(row.timestamp),}));diff --git a/packages/shared/src/server/repositories/scores.ts b/packages/shared/src/server/repositories/scores.tsindex 82c68a56c8..761e28954a 100644--- a/packages/shared/src/server/repositories/scores.ts+++ b/packages/shared/src/server/repositories/scores.ts@@ -225,7 +225,9 @@ const formatMetadataSelect = (includeHasMetadata: boolean,) => {return [- !excludeMetadata ? "*" : "* EXCEPT (metadata)",+ !excludeMetadata+ ? "* EXCEPT (evaluator_id, evaluation_rule_id)"+ : "* EXCEPT (metadata, evaluator_id, evaluation_rule_id)",includeHasMetadata? "length(mapKeys(s.metadata)) > 0 AS has_metadata": null,@@ -644,14 +646,7 @@ export const getScoresForObservations = async <includeHasMetadata = false,} = props;- const select = [- !excludeMetadata ? "*" : "* EXCEPT (metadata)",- includeHasMetadata- ? "length(mapKeys(s.metadata)) > 0 AS has_metadata"- : null,- ]- .filter((s) => s != null)- .join(", ");+ const select = formatMetadataSelect(excludeMetadata, includeHasMetadata);const query = `select@@ -1559,7 +1554,7 @@ const getScoresUiGenericFromEvents = async <T>(props: {s.source,s.data_type,s.comment,- s.metadata['evaluator_id'] AS evaluator_id,+ s.evaluator_id,${excludeMetadata ? "" : "s.metadata,"}s.trace_id,s.session_id,diff --git a/packages/shared/src/server/tableMappings/mapEventsTable.ts b/packages/shared/src/server/tableMappings/mapEventsTable.tsindex cee429475b..96faf84172 100644--- a/packages/shared/src/server/tableMappings/mapEventsTable.ts+++ b/packages/shared/src/server/tableMappings/mapEventsTable.ts@@ -2,7 +2,6 @@// The frontend only sends the column names to the backend. This needs to be changed in the future to send column IDs.import { UiColumnMappings } from "../../tableDefinitions";-import { EvalExecutionMetadataKey } from "../../features/evals/evalExecutionMetadata";import {eventsTableHasParentObservationSql,eventsTableIsRootObservationSql,@@ -220,13 +219,13 @@ export const eventsTableNativeUiColumnDefinitions: UiColumnMappings = [uiTableName: "Evaluator ID",uiTableId: "evaluatorId",clickhouseTableName: "events_proto",- clickhouseSelect: `arrayElement(e.metadata_values, indexOf(e.metadata_names, '${EvalExecutionMetadataKey.EVALUATOR_ID}'))`,+ clickhouseSelect: "e.evaluator_id",},{uiTableName: "Rule ID",uiTableId: "ruleId",clickhouseTableName: "events_proto",- clickhouseSelect: `if(notEmpty(arrayElement(e.metadata_values, indexOf(e.metadata_names, '${EvalExecutionMetadataKey.EVALUATION_RULE_ID}'))), arrayElement(e.metadata_values, indexOf(e.metadata_names, '${EvalExecutionMetadataKey.EVALUATION_RULE_ID}')), arrayElement(e.metadata_values, indexOf(e.metadata_names, '${EvalExecutionMetadataKey.JOB_CONFIGURATION_ID}')))`,+ clickhouseSelect: "e.evaluation_rule_id",},{uiTableName: "Version",diff --git a/packages/shared/src/server/tableMappings/mapScoresTable.ts b/packages/shared/src/server/tableMappings/mapScoresTable.tsindex cd2ddbb7a6..5b521d43ab 100644--- a/packages/shared/src/server/tableMappings/mapScoresTable.ts+++ b/packages/shared/src/server/tableMappings/mapScoresTable.ts@@ -63,8 +63,15 @@ export const scoresTableUiColumnDefinitions: UiColumnMappings = [uiTableName: "Evaluator ID",uiTableId: "evaluatorId",clickhouseTableName: "scores",- clickhouseSelect:- "coalesce(nullIf(s.metadata['evaluator_id'], ''), s.metadata['job_configuration_id'])",+ clickhouseSelect: "evaluator_id",+ queryPrefix: "s",+ },+ {+ uiTableName: "Rule ID",+ uiTableId: "ruleId",+ clickhouseTableName: "scores",+ clickhouseSelect: "evaluation_rule_id",+ queryPrefix: "s",},{uiTableName: "Value",diff --git a/packages/shared/src/tableDefinitions/scoresTable.ts b/packages/shared/src/tableDefinitions/scoresTable.tsindex 3c1aca903c..91fb5b47f4 100644--- a/packages/shared/src/tableDefinitions/scoresTable.ts+++ b/packages/shared/src/tableDefinitions/scoresTable.ts@@ -70,6 +70,14 @@ export const scoresTableCols: ColumnDefinition[] = [internal: "",options: [],},+ {+ name: "Rule ID",+ id: "ruleId",+ type: "stringOptions",+ // ClickHouse-only virtual column; see mapScoresTable.ts.+ internal: "",+ options: [],+ },{name: "Metadata",id: "metadata",diff --git a/web/src/__tests__/server/scores-trpc.servertest.ts b/web/src/__tests__/server/scores-trpc.servertest.tsindex d26bea4dcd..6451fb9b1f 100644--- a/web/src/__tests__/server/scores-trpc.servertest.ts+++ b/web/src/__tests__/server/scores-trpc.servertest.ts@@ -195,7 +195,7 @@ describe("scores trpc", () => {);});- it("filters evaluator scores by recorded evaluator and legacy rule metadata", async () => {+ it("filters evaluator scores only by recorded evaluator metadata", async () => {const [evaluator, otherEvaluator] = await Promise.all(["Evaluator", "Other evaluator"].map((name) =>prisma.evaluator.create({@@ -256,13 +256,13 @@ describe("scores trpc", () => {]);expect(new Set(scores.scores.map(({ id }) => id))).toEqual(- new Set([directScore.id, legacyScore.id]),+ new Set([directScore.id]),);expect(new Set(eventScores.scores.map(({ id }) => id))).toEqual(- new Set([directScore.id, legacyScore.id]),+ new Set([directScore.id]),);- expect(count.totalCount).toBe(2);- expect(eventCount.totalCount).toBe(2);+ expect(count.totalCount).toBe(1);+ expect(eventCount.totalCount).toBe(1);});it("does not match empty boolean representations when filtering boolean values", async () => {diff --git a/web/src/__tests__/server/unit/evaluator-v2-test-evaluator.servertest.ts b/web/src/__tests__/server/unit/evaluator-v2-test-evaluator.servertest.tsindex c7b1e6177f..b2e171da63 100644--- a/web/src/__tests__/server/unit/evaluator-v2-test-evaluator.servertest.ts+++ b/web/src/__tests__/server/unit/evaluator-v2-test-evaluator.servertest.ts@@ -166,7 +166,12 @@ describe("testEvaluator", () => {}),],trace: expect.objectContaining({+ evaluationContext: {+ evaluatorId: "evaluator-1",+ evaluatorExecutionIsTest: true,+ },metadata: expect.objectContaining({+ evaluator_id: "evaluator-1",evaluator_test: "true",target_trace_id: "trace-1",target_observation_id: "observation-1",@@ -280,7 +285,12 @@ describe("testEvaluator", () => {},{ var: "experimentItemMetadata", value: { source: "test" } },],+ evaluationContext: {+ evaluatorId: "evaluator-1",+ evaluatorExecutionIsTest: true,+ },metadata: expect.objectContaining({+ evaluator_id: "evaluator-1",evaluator_test: "true",target_trace_id: "trace-1",target_observation_id: "observation-1",diff --git a/web/src/features/evals/server/codeEvalTestRun.ts b/web/src/features/evals/server/codeEvalTestRun.tsindex 8b66a1d02a..e094161f09 100644--- a/web/src/features/evals/server/codeEvalTestRun.ts+++ b/web/src/features/evals/server/codeEvalTestRun.ts@@ -1,6 +1,6 @@import {applyCommentFilters,- buildEvalExecutionMetadata,+ buildEvalExecutionData,createW3CTraceId,extractObservationVariables,getEventsStreamForEval,@@ -166,7 +166,6 @@ export async function runCodeEvalTestForEvaluationRule(params: {sourceCodeLanguage: version.sourceCodeLanguage,},evaluatorMetadata: {- evaluator_id: evaluator.id,evaluator_version: version.version,},});@@ -244,7 +243,6 @@ async function runCodeEvalTestForObservation(params: {evaluator: codeTemplate,version: codeTemplate,evaluatorMetadata: {- evaluator_id: codeTemplate.id,evaluator_version: codeTemplate.version,},});@@ -288,20 +286,20 @@ async function runCodeEvalTestForObservationWithEvaluator(params: {});const executionTraceId = createW3CTraceId();const traceName = `Test evaluator: ${params.evaluator.name}`;+ const evaluationData = buildEvalExecutionData({+ type: "TEST",+ evaluatorId: params.evaluator.id,+ targetTraceId: params.observation.trace_id,+ targetObservationId: params.observation.span_id,+ });const executionMetadata = {dispatcher_name: dispatcher.name,code_eval_runtime: params.version.sourceCodeLanguage,...params.evaluatorMetadata,- ...buildEvalExecutionMetadata({- type: "TEST",- evaluatorId: params.evaluator.id,- targetTraceId: params.observation.trace_id,- targetObservationId: params.observation.span_id,- }),+ ...evaluationData.executionMetadata,score_name: params.scoreName,target_object: params.target,};-const dispatchOutcome = await runCodeBasedEvaluationDispatch({dispatcher,organizationId: params.orgId,@@ -314,6 +312,7 @@ async function runCodeEvalTestForObservationWithEvaluator(params: {hasExperimentContext: Boolean(params.observation.experiment_id),traceName,metadata: executionMetadata,+ evaluationContext: evaluationData.evaluationContext,writeTrace: writeInternalTraceViaOtelIngestion,});diff --git a/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.clienttest.ts b/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.clienttest.tsindex e54ececc06..e11accf48c 100644--- a/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.clienttest.ts+++ b/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.clienttest.ts@@ -11,14 +11,13 @@ import {} from "./evaluatorScoresUrl";describe("evaluatorScoresUrl", () => {- it("filters code evaluator scores by evaluator and assigned rule IDs", () => {+ it("filters code evaluator scores by evaluator ID", () => {const url = new URL(evaluatorScoresUrl("project-1","evaluator-id","Code evaluator",EvalTemplateTypeEnum.CODE,- ["rule-1", "rule-2"],),"https://langfuse.local",);@@ -30,7 +29,7 @@ describe("evaluatorScoresUrl", () => {column: "evaluatorId",type: "stringOptions",operator: "any of",- value: ["evaluator-id", "rule-1", "rule-2"],+ value: ["evaluator-id"],},{column: "source",@@ -48,7 +47,6 @@ describe("evaluatorScoresUrl", () => {"evaluator-id","Correctness",EvalTemplateTypeEnum.LLM_AS_JUDGE,- ["rule-1"],),"https://langfuse.local",);diff --git a/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.ts b/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.tsindex 8116310b46..486b2f2745 100644--- a/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.ts+++ b/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.ts@@ -11,7 +11,6 @@ export function evaluatorScoresUrl(evaluatorId: string,evaluatorName: string,evaluatorType: EvalTemplateType,- assignedRuleIds: string[],) {const evaluatorFilter: FilterState[number] =evaluatorType === EvalTemplateTypeEnum.CODE@@ -19,7 +18,7 @@ export function evaluatorScoresUrl(column: "evaluatorId",type: "stringOptions",operator: "any of",- value: [...new Set([evaluatorId, ...assignedRuleIds])],+ value: [evaluatorId],}: {column: "name",diff --git a/web/src/features/evals/v2/pages/EvaluatorsPage.tsx b/web/src/features/evals/v2/pages/EvaluatorsPage.tsxindex 0ee3a9bd2e..f7626a24cb 100644--- a/web/src/features/evals/v2/pages/EvaluatorsPage.tsx+++ b/web/src/features/evals/v2/pages/EvaluatorsPage.tsx@@ -494,7 +494,6 @@ export default function EvaluatorsPage() {row.original.id,row.original.name,row.original.type,- row.original.assignedRuleIds,),)}diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorService.ts b/web/src/features/evals/v2/server/evaluators/evaluatorService.tsindex fcfde562a4..d9621bb035 100644--- a/web/src/features/evals/v2/server/evaluators/evaluatorService.ts+++ b/web/src/features/evals/v2/server/evaluators/evaluatorService.ts@@ -259,8 +259,8 @@ export class EvaluatorService {projectId: params.projectId,evaluatorIds: params.evaluatorIds,});- // Prompt-experiment executions do not always carry evaluator_id metadata,- // but all evaluator execution paths use this trace-name convention.+ // Prompt-experiment executions do not always carry an evaluator id, but+ // all evaluator execution paths use this trace-name convention.const evaluatorIdsByTraceName = new Map<string, string[]>();for (const evaluator of evaluators) {const traceName = `Execute evaluator: ${evaluator.name}`;diff --git a/web/src/features/evals/v2/server/evaluators/testEvaluator.ts b/web/src/features/evals/v2/server/evaluators/testEvaluator.tsindex 82516483af..a83d722343 100644--- a/web/src/features/evals/v2/server/evaluators/testEvaluator.ts+++ b/web/src/features/evals/v2/server/evaluators/testEvaluator.ts@@ -3,7 +3,7 @@ import {observationVariableMappingList,} from "@langfuse/shared";import {- buildEvalExecutionMetadata,+ buildEvalExecutionData,compileLangfuseMediaMessages,createW3CTraceId,DefaultEvalModelService,@@ -62,7 +62,7 @@ export async function testEvaluator(params: {observation,variableMapping,});- const metadata = buildEvalExecutionMetadata({+ const executionData = buildEvalExecutionData({type: "TEST",evaluatorId:params.includeEvaluatorLink === false ? null : params.evaluatorId,@@ -78,14 +78,14 @@ export async function testEvaluator(params: {evaluatorId: params.evaluatorId,definition: params.definition,variables,- metadata,+ ...executionData,}): await testLlmEvaluator({projectId: params.projectId,evaluatorId: params.evaluatorId,definition: params.definition,variables,- metadata,+ ...executionData,});return { ...result, durationMs: Date.now() - startedAt };@@ -96,7 +96,10 @@ async function testLlmEvaluator(params: {evaluatorId: string;definition: Extract<NormalizedEvaluatorDefinition, { type: "LLM_AS_JUDGE" }>;variables: ExtractedVariable[];- metadata: ReturnType<typeof buildEvalExecutionMetadata>;+ executionMetadata: Record<string, string>;+ evaluationContext: ReturnType<+ typeof buildEvalExecutionData+ >["evaluationContext"];}) {const modelConfig = await DefaultEvalModelService.fetchValidModelConfig(params.projectId,@@ -149,7 +152,8 @@ async function testLlmEvaluator(params: {traceId: executionTraceId,traceName: "Test evaluator",environment: LangfuseInternalTraceEnvironment.LLMJudge,- metadata: params.metadata,+ metadata: params.executionMetadata,+ evaluationContext: params.evaluationContext,},});estimatedCostUsd = await calculateTestRunCost({@@ -225,7 +229,10 @@ async function testCodeEvaluator(params: {evaluatorId: string;definition: Extract<NormalizedEvaluatorDefinition, { type: "CODE" }>;variables: ExtractedVariable[];- metadata: ReturnType<typeof buildEvalExecutionMetadata>;+ executionMetadata: Record<string, string>;+ evaluationContext: ReturnType<+ typeof buildEvalExecutionData+ >["evaluationContext"];}) {const dispatcher = resolveConfiguredCodeEvalDispatcher();if (!dispatcher) {@@ -236,7 +243,6 @@ async function testCodeEvaluator(params: {}const executionTraceId = createW3CTraceId();-return runCodeBasedEvaluationDispatch({dispatcher,organizationId: params.orgId,@@ -247,6 +253,7 @@ async function testCodeEvaluator(params: {version: params.definition,extractedVariables: params.variables,traceName: "Test evaluator",- metadata: params.metadata,+ metadata: params.executionMetadata,+ evaluationContext: params.evaluationContext,});}diff --git a/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.test.ts b/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.test.tsindex c8bc9a772c..a323607060 100644--- a/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.test.ts+++ b/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.test.ts@@ -44,6 +44,23 @@ vi.mock("@langfuse/shared/src/server", async (importOriginal) => {import { executeCodeBasedEvaluation } from "./executeCodeBasedEvaluation";+const evaluationContext = {+ evaluatorId: "evaluator-1",+ evaluationRuleId: "config-1",+ evaluatorExecutionIsTest: false,+};++const executeCodeBasedEvaluationWithContext = (+ params: Omit<+ Parameters<typeof executeCodeBasedEvaluation>[0],+ "evaluationContext"+ >,+) =>+ executeCodeBasedEvaluation({+ ...params,+ evaluationContext,+ });+describe("executeCodeBasedEvaluation", () => {beforeEach(() => {vi.clearAllMocks();@@ -63,7 +80,7 @@ describe("executeCodeBasedEvaluation", () => {],});- const result = await executeCodeBasedEvaluation({+ const result = await executeCodeBasedEvaluationWithContext({projectId: "project-1",organizationId: "org-1",evaluatorId: "evaluator-1",@@ -177,7 +194,7 @@ describe("executeCodeBasedEvaluation", () => {],});- const result = await executeCodeBasedEvaluation({+ const result = await executeCodeBasedEvaluationWithContext({projectId: "project-1",organizationId: "org-1",job: {@@ -228,7 +245,7 @@ describe("executeCodeBasedEvaluation", () => {scores: [{ name: "score", value: 1, dataType: "BOOLEAN" }],});- await executeCodeBasedEvaluation({+ await executeCodeBasedEvaluationWithContext({projectId: "project-1",organizationId: "org-1",job: {@@ -276,7 +293,7 @@ describe("executeCodeBasedEvaluation", () => {scores: [{ name: "score", value: 1, dataType: "BOOLEAN" }],});- await executeCodeBasedEvaluation({+ await executeCodeBasedEvaluationWithContext({projectId: "project-1",organizationId: "org-1",job: {@@ -320,7 +337,7 @@ describe("executeCodeBasedEvaluation", () => {scores: [{ name: "score", value: 1, dataType: "BOOLEAN" }],});- await executeCodeBasedEvaluation({+ await executeCodeBasedEvaluationWithContext({projectId: "project-1",organizationId: "org-1",job: {@@ -372,7 +389,7 @@ describe("executeCodeBasedEvaluation", () => {scores: [{ name: "score", value: 1, dataType: "BOOLEAN" }],});- await executeCodeBasedEvaluation({+ await executeCodeBasedEvaluationWithContext({projectId: "project-1",organizationId: "org-1",job: {@@ -427,7 +444,7 @@ describe("executeCodeBasedEvaluation", () => {mocks.writeInternalTrace.mockRejectedValue(new Error("trace write failed"));await expect(- executeCodeBasedEvaluation({+ executeCodeBasedEvaluationWithContext({projectId: "project-1",organizationId: "org-1",job: {@@ -464,7 +481,7 @@ describe("executeCodeBasedEvaluation", () => {});mocks.dispatcher.dispatch.mockRejectedValue(error);- const promise = executeCodeBasedEvaluation({+ const promise = executeCodeBasedEvaluationWithContext({projectId: "project-1",organizationId: "org-1",job: {@@ -538,7 +555,7 @@ describe("executeCodeBasedEvaluation", () => {parseDispatchResult({ score: 1 }),);- const promise = executeCodeBasedEvaluation({+ const promise = executeCodeBasedEvaluationWithContext({projectId: "project-1",organizationId: "org-1",job: {@@ -600,7 +617,7 @@ describe("executeCodeBasedEvaluation", () => {});mocks.dispatcher.dispatch.mockRejectedValue(error);- const promise = executeCodeBasedEvaluation({+ const promise = executeCodeBasedEvaluationWithContext({projectId: "project-1",organizationId: "org-1",job: {@@ -701,7 +718,7 @@ describe("executeCodeBasedEvaluation", () => {);mocks.dispatcher.dispatch.mockRejectedValue(error);- const promise = executeCodeBasedEvaluation({+ const promise = executeCodeBasedEvaluationWithContext({projectId: "project-1",organizationId: "org-1",job: {diff --git a/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.ts b/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.tsindex d818e8e164..19e392c482 100644--- a/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.ts+++ b/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.ts@@ -1,5 +1,8 @@import { type JobConfiguration, type JobExecution } from "@prisma/client";-import { type EvalTemplateCodeBased } from "@langfuse/shared";+import {+ type EvalExecutionContext,+ type EvalTemplateCodeBased,+} from "@langfuse/shared";import {CodeEvalExecutionError,instrumentAsync,@@ -24,6 +27,7 @@ export async function executeCodeBasedEvaluation(params: {extractedVariables: ExtractedVariable[];hasExperimentContext?: boolean;executionMetadata: Record<string, string>;+ evaluationContext: EvalExecutionContext;// Unused by code-based eval; the shared observation processor passes it.deps?: EvalExecutionDeps;}): Promise<EvalExecutionResult> {@@ -70,6 +74,7 @@ export async function executeCodeBasedEvaluation(params: {hasExperimentContext: params.hasExperimentContext ?? false,traceName: `Execute evaluator: ${params.template.name}`,metadata: executionMetadata,+ evaluationContext: params.evaluationContext,// Publish via the OTel ingestion pipeline (like LLM-as-a-judge) so the// trace reaches the legacy tables too in dual write mode — a direct// events-table write alone 404s in the trace detail view.@@ -86,6 +91,7 @@ export async function executeCodeBasedEvaluation(params: {scores: dispatchOutcome.scores,executionTraceId,metadata: executionMetadata,+ evaluationContext: params.evaluationContext,};},);diff --git a/worker/src/features/evaluation/evalCompletion.ts b/worker/src/features/evaluation/evalCompletion.tsindex 3a73274944..910e3a2a16 100644--- a/worker/src/features/evaluation/evalCompletion.ts+++ b/worker/src/features/evaluation/evalCompletion.ts@@ -1,4 +1,5 @@import { JobExecutionStatus } from "@prisma/client";+import type { EvalExecutionContext } from "@langfuse/shared";import {logger,traceException,@@ -16,6 +17,7 @@ export type EvalExecutionResult = {scores: CodeEvalScoreWithName[];executionTraceId: string;metadata: Record<string, string>;+ evaluationContext: EvalExecutionContext;};export async function completeEvalExecution({@@ -43,6 +45,7 @@ export async function completeEvalExecution({environment,executionTraceId: result.executionTraceId,executionMetadata: result.metadata,+ evaluationContext: result.evaluationContext,});const [firstScorePayload] = scoreWritePayloads;diff --git a/worker/src/features/evaluation/evalExecutionDeps.ts b/worker/src/features/evaluation/evalExecutionDeps.tsindex 89fdc76d77..1f56973181 100644--- a/worker/src/features/evaluation/evalExecutionDeps.ts+++ b/worker/src/features/evaluation/evalExecutionDeps.ts@@ -1,6 +1,7 @@import { randomUUID } from "crypto";import { z } from "zod";import { JobExecutionStatus } from "@prisma/client";+import type { EvalExecutionContext } from "@langfuse/shared";import { prisma } from "@langfuse/shared/src/db";import {buildEventBucketPrefix,@@ -63,6 +64,7 @@ interface LLMCallParams {traceName: string;environment: string;metadata: Record<string, unknown>;+ evaluationContext?: EvalExecutionContext;};}@@ -282,6 +284,7 @@ export function createProductionEvalExecutionDeps(): EvalExecutionDeps {traceName: params.traceSinkParams.traceName,environment: params.traceSinkParams.environment,metadata: params.traceSinkParams.metadata,+ evaluationContext: params.traceSinkParams.evaluationContext,eventsWriter: createInternalEventsWriter(),},});diff --git a/worker/src/features/evaluation/evalScoreEvent.ts b/worker/src/features/evaluation/evalScoreEvent.tsindex de072f4799..abc15b9de6 100644--- a/worker/src/features/evaluation/evalScoreEvent.ts+++ b/worker/src/features/evaluation/evalScoreEvent.ts@@ -1,5 +1,5 @@import { randomUUID } from "crypto";-import { ScoreSourceEnum } from "@langfuse/shared";+import { ScoreSourceEnum, type EvalExecutionContext } from "@langfuse/shared";import {buildDeterministicEvalScoreIds,eventTypes,@@ -10,7 +10,12 @@ import {export type EvalScoreWritePayload = {eventId: string;scoreId: string;- event: ScoreEventType;+ event: ScoreEventType & {+ body: ScoreEventType["body"] & {+ evaluatorId?: string;+ evaluationRuleId?: string;+ };+ };};export function buildEvalScoreWritePayloads(params: {@@ -21,6 +26,7 @@ export function buildEvalScoreWritePayloads(params: {environment: string;executionTraceId: string;executionMetadata: Record<string, string>;+ evaluationContext: EvalExecutionContext;}): EvalScoreWritePayload[] {const scoreIds = buildDeterministicEvalScoreIds({scores: params.scores,@@ -30,7 +36,6 @@ export function buildEvalScoreWritePayloads(params: {return params.scores.map((score, index) => {const eventId = randomUUID();const scoreId = scoreIds[index]!;-return {eventId,scoreId,@@ -48,13 +53,15 @@ export function buildEvalScoreWritePayloads(params: {...(score.metadata ?? {}),...params.executionMetadata,},+ evaluatorId: params.evaluationContext.evaluatorId,+ evaluationRuleId: params.evaluationContext.evaluationRuleId,configId: score.configId,source: ScoreSourceEnum.EVAL,environment: params.environment,executionTraceId: params.executionTraceId,value: score.value,dataType: score.dataType,- } as ScoreEventType["body"],+ } as EvalScoreWritePayload["event"]["body"],},};});diff --git a/worker/src/features/evaluation/evalService.ts b/worker/src/features/evaluation/evalService.tsindex 812b0e7aed..7f8be1ec27 100644--- a/worker/src/features/evaluation/evalService.ts+++ b/worker/src/features/evaluation/evalService.ts@@ -32,7 +32,7 @@ import {DatasetRunItemUpsertEventType,classifyEvaluatorLlmError,blockEvaluator,- buildEvalExecutionMetadata,+ buildEvalExecutionData,EvaluatorBlockSource,executeLlmEvaluator,type CodeEvalScoreWithName,@@ -65,6 +65,7 @@ import {type EvalOutputResult,extractValueFromObject,validateEvaluatorFiltersForTarget,+ type EvalExecutionContext,} from "@langfuse/shared";import { env } from "../../env";import { prisma } from "@langfuse/shared/src/db";@@ -944,6 +945,7 @@ export async function runLLMAsJudgeEvaluation({template,extractedVariables,executionMetadata,+ evaluationContext,deps,evaluatorId,}: {@@ -954,6 +956,7 @@ export async function runLLMAsJudgeEvaluation({template: EvalTemplateLlmAsAJudge;extractedVariables: ExtractedVariable[];executionMetadata: Record<string, string>;+ evaluationContext: EvalExecutionContext;deps: EvalExecutionDeps;/*** Evaluator v2 identity, when the execution came from an evaluation rule.@@ -1141,9 +1144,8 @@ export async function runLLMAsJudgeEvaluation({traceId: executionTraceId,traceName: `Execute evaluator: ${template.name}`,environment: LangfuseInternalTraceEnvironment.LLMJudge,- metadata: {- ...executionMetadata,- },+ metadata: executionMetadata,+ evaluationContext,},});llmSpan.setAttribute("eval.llm.outcome", "success");@@ -1223,6 +1225,7 @@ export async function runLLMAsJudgeEvaluation({scores,executionTraceId,metadata: executionMetadata,+ evaluationContext,};},);@@ -1268,7 +1271,7 @@ function toNormalizedScores(params: {export async function executeLLMAsJudgeEvaluation(params: Omit<Parameters<typeof runLLMAsJudgeEvaluation>[0],- "deps" | "executionMetadata"+ "deps" | "executionMetadata" | "evaluationContext"> & {environment: string;deps?: EvalExecutionDeps;@@ -1278,7 +1281,7 @@ export async function executeLLMAsJudgeEvaluation(},): Promise<void> {const deps = params.deps ?? createProductionEvalExecutionDeps();- const executionMetadata = buildEvalExecutionMetadata({+ const executionData = buildEvalExecutionData({type: "JOB",jobExecutionId: params.jobExecutionId,jobConfigurationId: params.job.jobConfigurationId,@@ -1297,7 +1300,7 @@ export async function executeLLMAsJudgeEvaluation(const result = await runLLMAsJudgeEvaluation({...params,deps,- executionMetadata,+ ...executionData,});await completeEvalExecution({diff --git a/worker/src/features/evaluation/evaluationHelpers.test.ts b/worker/src/features/evaluation/evaluationHelpers.test.tsindex ccbece6444..446852bda4 100644--- a/worker/src/features/evaluation/evaluationHelpers.test.ts+++ b/worker/src/features/evaluation/evaluationHelpers.test.ts@@ -11,7 +11,7 @@ import {validateEvalOutputResult,} from "@langfuse/shared";import {- buildEvalExecutionMetadata,+ buildEvalExecutionData,type EvaluatorLlmErrorClassification,} from "@langfuse/shared/src/server";import { parseDispatchResult } from "../../../../packages/shared/src/server/evals/codeEvalDispatcherTypes";@@ -267,7 +267,7 @@ describe("evaluation helpers", () => {});});- describe("buildEvalExecutionMetadata", () => {+ describe("buildEvalExecutionData", () => {it("should include all provided fields", () => {const params = {type: "JOB" as const,@@ -278,14 +278,20 @@ describe("evaluation helpers", () => {targetDatasetItemId: "dataset-def",};- const result = buildEvalExecutionMetadata(params);+ const result = buildEvalExecutionData(params);expect(result).toEqual({- job_execution_id: "exec-123",- job_configuration_id: "config-456",- target_trace_id: "trace-789",- target_observation_id: "obs-abc",- target_dataset_item_id: "dataset-def",+ executionMetadata: {+ job_execution_id: "exec-123",+ job_configuration_id: "config-456",+ target_trace_id: "trace-789",+ target_observation_id: "obs-abc",+ target_dataset_item_id: "dataset-def",+ },+ evaluationContext: {+ evaluationRuleId: "config-456",+ evaluatorExecutionIsTest: false,+ },});});@@ -299,15 +305,27 @@ describe("evaluation helpers", () => {targetDatasetItemId: null,};- const result = buildEvalExecutionMetadata(params);+ const result = buildEvalExecutionData(params);expect(result).toEqual({- job_execution_id: "exec-123",- job_configuration_id: "config-456",+ executionMetadata: {+ job_execution_id: "exec-123",+ job_configuration_id: "config-456",+ },+ evaluationContext: {+ evaluationRuleId: "config-456",+ evaluatorExecutionIsTest: false,+ },});- expect(Object.keys(result)).not.toContain("target_trace_id");- expect(Object.keys(result)).not.toContain("target_observation_id");- expect(Object.keys(result)).not.toContain("target_dataset_item_id");+ expect(Object.keys(result.executionMetadata)).not.toContain(+ "target_trace_id",+ );+ expect(Object.keys(result.executionMetadata)).not.toContain(+ "target_observation_id",+ );+ expect(Object.keys(result.executionMetadata)).not.toContain(+ "target_dataset_item_id",+ );});});@@ -513,6 +531,11 @@ describe("evaluation helpers", () => {});describe("buildEvalScoreWritePayloads", () => {+ const evaluationContext = {+ evaluationRuleId: "rule-1",+ evaluatorExecutionIsTest: false,+ };+it("should build stable code eval score IDs when different score names reorder", () => {const originalPayloads = buildEvalScoreWritePayloads({scores: [@@ -533,6 +556,7 @@ describe("evaluation helpers", () => {environment: "production",executionTraceId: "exec-trace-789",executionMetadata: { job_execution_id: "job-1" },+ evaluationContext,});const reorderedPayloads = buildEvalScoreWritePayloads({scores: [@@ -553,6 +577,7 @@ describe("evaluation helpers", () => {environment: "production",executionTraceId: "exec-trace-789",executionMetadata: { job_execution_id: "job-1" },+ evaluationContext,});expect(originalPayloads[0].scoreId).toBe(reorderedPayloads[1].scoreId);@@ -584,6 +609,7 @@ describe("evaluation helpers", () => {environment: "production",executionTraceId: "exec-trace-789",executionMetadata: { job_execution_id: "job-1" },+ evaluationContext,});const scoreIds = result.map((payload) => payload.scoreId);@@ -639,6 +665,7 @@ describe("evaluation helpers", () => {environment: "production",executionTraceId: "exec-trace-789",executionMetadata: { job_execution_id: "job-1" },+ evaluationContext,});expect(result.map((payload) => payload.scoreId)).toEqual(@@ -670,6 +697,7 @@ describe("evaluation helpers", () => {environment: "production",executionTraceId: "exec-trace-789",executionMetadata: { job_execution_id: "job-1" },+ evaluationContext,});expect(result).toHaveLength(1);@@ -706,6 +734,7 @@ describe("evaluation helpers", () => {environment: "production",executionTraceId: "exec-trace-789",executionMetadata: { job_execution_id: "job-1" },+ evaluationContext,});expect(result).toHaveLength(1);@@ -753,6 +782,7 @@ describe("evaluation helpers", () => {environment: "production",executionTraceId: "exec-trace-789",executionMetadata: { job_execution_id: "job-1" },+ evaluationContext,});expect(result).toHaveLength(2);@@ -798,6 +828,11 @@ describe("evaluation helpers", () => {job_execution_id: "job-1",dispatcher_name: "test-dispatcher",},+ evaluationContext: {+ evaluatorId: "evaluator-1",+ evaluationRuleId: "legacy-rule-1",+ evaluatorExecutionIsTest: false,+ },});expect(result[0].event.body.metadata).toEqual({@@ -806,10 +841,18 @@ describe("evaluation helpers", () => {job_execution_id: "job-1",dispatcher_name: "test-dispatcher",});+ expect(result[0].event.body).toMatchObject({+ evaluatorId: "evaluator-1",+ evaluationRuleId: "legacy-rule-1",+ });expect(result[1].event.body.metadata).toEqual({job_execution_id: "job-1",dispatcher_name: "test-dispatcher",});+ expect(result[1].event.body).toMatchObject({+ evaluatorId: "evaluator-1",+ evaluationRuleId: "legacy-rule-1",+ });});});diff --git a/worker/src/features/evaluation/executeLLMAsJudgeEvaluation.test.ts b/worker/src/features/evaluation/executeLLMAsJudgeEvaluation.test.tsindex e1986b0d9b..2ca85d6a37 100644--- a/worker/src/features/evaluation/executeLLMAsJudgeEvaluation.test.ts+++ b/worker/src/features/evaluation/executeLLMAsJudgeEvaluation.test.ts@@ -525,9 +525,12 @@ describe("executeLLMAsJudgeEvaluation", () => {traceId: expect.any(String),traceName: `Execute evaluator: ${mockEvalTemplate.name}`,environment: "langfuse-llm-as-a-judge",+ evaluationContext: {+ evaluationRuleId: mockJobExecution.jobConfigurationId,+ evaluatorExecutionIsTest: false,+ },metadata: expect.objectContaining({job_execution_id: jobExecutionId,- job_configuration_id: mockJobExecution.jobConfigurationId,target_trace_id: mockJobExecution.jobInputTraceId,}),}),@@ -844,9 +847,8 @@ describe("executeLLMAsJudgeEvaluation", () => {createExecutionParams({ job: fullJob, deps }),);- const expectedMetadata = {+ const expectedTraceMetadata = {job_execution_id: jobExecutionId,- job_configuration_id: fullJob.jobConfigurationId,target_trace_id: fullJob.jobInputTraceId,target_observation_id: "obs-full",target_dataset_item_id: "dataset-full",@@ -855,7 +857,11 @@ describe("executeLLMAsJudgeEvaluation", () => {expect(callLLM).toHaveBeenCalledWith(expect.objectContaining({traceSinkParams: expect.objectContaining({- metadata: expect.objectContaining(expectedMetadata),+ evaluationContext: {+ evaluationRuleId: fullJob.jobConfigurationId,+ evaluatorExecutionIsTest: false,+ },+ metadata: expect.objectContaining(expectedTraceMetadata),}),}),);@@ -864,7 +870,13 @@ describe("executeLLMAsJudgeEvaluation", () => {expect.objectContaining({event: expect.objectContaining({body: expect.objectContaining({- metadata: expect.objectContaining(expectedMetadata),+ evaluationRuleId: fullJob.jobConfigurationId,+ metadata: expect.objectContaining({+ job_execution_id: jobExecutionId,+ target_trace_id: fullJob.jobInputTraceId,+ target_observation_id: "obs-full",+ target_dataset_item_id: "dataset-full",+ }),}),}),}),@@ -897,12 +909,14 @@ describe("executeLLMAsJudgeEvaluation", () => {name: mockJobConfiguration.scoreName,value: 0.88,comment: "Well structured response",+ configId: undefined,source: "EVAL",environment: "production",executionTraceId: expect.any(String),+ evaluatorId: undefined,+ evaluationRuleId: mockJobExecution.jobConfigurationId,metadata: expect.objectContaining({job_execution_id: jobExecutionId,- job_configuration_id: mockJobExecution.jobConfigurationId,target_trace_id: mockJobExecution.jobInputTraceId,}),dataType: "NUMERIC",diff --git a/worker/src/features/evaluation/observationEval/__tests__/observationEval.e2e.test.ts b/worker/src/features/evaluation/observationEval/__tests__/observationEval.e2e.test.tsindex 3d03a7df41..0eedf1e10e 100644--- a/worker/src/features/evaluation/observationEval/__tests__/observationEval.e2e.test.ts+++ b/worker/src/features/evaluation/observationEval/__tests__/observationEval.e2e.test.ts@@ -88,6 +88,9 @@ const mockEvalExecutionResult = {],executionTraceId: "trace-123",metadata: {},+ evaluationContext: {+ evaluatorExecutionIsTest: false,+ },};describe("Observation Eval E2E Pipeline", () => {diff --git a/worker/src/features/evaluation/observationEval/__tests__/observationEvalProcessor.test.ts b/worker/src/features/evaluation/observationEval/__tests__/observationEvalProcessor.test.tsindex cf9152925e..5136d8d892 100644--- a/worker/src/features/evaluation/observationEval/__tests__/observationEvalProcessor.test.ts+++ b/worker/src/features/evaluation/observationEval/__tests__/observationEvalProcessor.test.ts@@ -99,6 +99,9 @@ const mockEvalExecutionResult = {],executionTraceId: "trace-123",metadata: {},+ evaluationContext: {+ evaluatorExecutionIsTest: false,+ },};describe("processObservationEval", () => {@@ -323,12 +326,14 @@ describe("processObservationEval", () => {}),template: expect.objectContaining({ id: version.id }),executionMetadata: expect.objectContaining({- evaluation_rule_id: rule.id,- job_configuration_id: rule.id,evaluation_rule_assignment_id: assignment.id,- evaluator_id: evaluator.id,evaluator_version_id: version.id,}),+ evaluationContext: {+ evaluatorId: evaluator.id,+ evaluationRuleId: rule.id,+ evaluatorExecutionIsTest: false,+ },// Pausing targets the evaluator, not the rule it ran for.evaluatorId: evaluator.id,}),@@ -492,9 +497,13 @@ describe("processObservationEval", () => {expect.objectContaining({config: expect.objectContaining({ id: evaluator.id }),executionMetadata: expect.objectContaining({- evaluator_id: evaluator.id,evaluator_version_id: version.id,}),+ evaluationContext: {+ evaluatorId: evaluator.id,+ evaluationRuleId: evaluator.id,+ evaluatorExecutionIsTest: false,+ },}),);});@@ -569,10 +578,11 @@ describe("processObservationEval", () => {expect.objectContaining({evaluatorId: evaluator.id,config: expect.objectContaining({ id: legacyConfig.id }),- executionMetadata: expect.objectContaining({- evaluation_rule_id: legacyConfig.id,- evaluator_id: evaluator.id,- }),+ evaluationContext: {+ evaluatorId: evaluator.id,+ evaluationRuleId: legacyConfig.id,+ evaluatorExecutionIsTest: false,+ },}),);});diff --git a/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts b/worker/src/features/evaluation/observationEval/observationEvalProcessor.tsindex 2fe0646144..81908385b0 100644--- a/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts+++ b/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts@@ -24,7 +24,7 @@ import {} from "@prisma/client";import { prisma, JobExecutionStatus } from "@langfuse/shared/src/db";import { UnrecoverableError } from "../../../errors/UnrecoverableError";-import { buildEvalExecutionMetadata } from "@langfuse/shared/src/server";+import { buildEvalExecutionData } from "@langfuse/shared/src/server";import {completeEvalExecution,type EvalExecutionResult,@@ -233,7 +233,7 @@ export async function processObservationEval(extractedVariables,hasExperimentContext: Boolean(observationData.experiment_id),environment: observationData.environment ?? DEFAULT_TRACE_ENVIRONMENT,- executionMetadata: buildEvalExecutionMetadata({+ ...buildEvalExecutionData({type: "JOB",jobExecutionId: event.jobExecutionId,jobConfigurationId: job.jobConfigurationId,diff --git a/worker/src/services/IngestionService/index.ts b/worker/src/services/IngestionService/index.tsindex 7f077af253..7900f8dab2 100644--- a/worker/src/services/IngestionService/index.ts+++ b/worker/src/services/IngestionService/index.ts@@ -304,6 +304,7 @@ export class IngestionService {const modelParameters = parseEventModelParameters(eventData.modelParameters,);+ const metadata = eventData.metadata ?? {};// Runs outside the modelName gate below so model-less events with provided// usage are still checked.@@ -338,7 +339,7 @@ export class IngestionService {projectId: eventData.projectId,pricingMatchAttributeValues: {modelParameters,- metadata: eventData.metadata,+ metadata,},observationRecord: {id: eventData.spanId,@@ -357,14 +358,11 @@ export class IngestionService {const now = this.getMicrosecondTimestamp();// Flatten raw metadata first (before stringification destroys nested structure)- const flattened = eventData.metadata- ? flattenJsonToPathArrays(eventData.metadata)- : { names: [], values: [] };+ const flattened = flattenJsonToPathArrays(metadata);const metadataNames = flattened.names;// Defensive: coerce null/undefined to empty string for Array(String) ClickHouse column.// Should not be required as convertValueToPlainJavascript() never returns null.const metadataValues = flattened.values.map((v) => v ?? "");-const eventRecord: EventRecordInsertType = {// Required identifiersid: eventData.spanId,@@ -440,6 +438,10 @@ export class IngestionService {// Metadatametadata_names: metadataNames,metadata_values: metadataValues,+ evaluator_id: eventData.evaluationContext?.evaluatorId,+ evaluation_rule_id: eventData.evaluationContext?.evaluationRuleId,+ evaluator_execution_is_test:+ eventData.evaluationContext?.evaluatorExecutionIsTest,// Source/instrumentation metadatasource: eventData.source,@@ -652,7 +654,14 @@ export class IngestionService {scoreId: entityId,projectId,});-+ const rawMetadata = scoreEvent.body.metadata+ ? convertJsonSchemaToRecord(scoreEvent.body.metadata)+ : {};+ const evaluationFields =+ scoreEvent.body as ScoreEventType["body"] & {+ evaluatorId?: string;+ evaluationRuleId?: string;+ };return {id: entityId,project_id: projectId,@@ -668,9 +677,9 @@ export class IngestionService {observation_id: validatedScore.observationId,config_id: validatedScore.configId,comment: validatedScore.comment,- metadata: scoreEvent.body.metadata- ? convertJsonSchemaToRecord(scoreEvent.body.metadata)- : {},+ metadata: convertRecordValuesToString(rawMetadata),+ evaluator_id: evaluationFields.evaluatorId,+ evaluation_rule_id: evaluationFields.evaluationRuleId,string_value: validatedScore.stringValue,long_string_value: validatedScore.longStringValue,execution_trace_id: validatedScore.executionTraceId,diff --git a/worker/src/services/IngestionService/tests/IngestionService.unit.test.ts b/worker/src/services/IngestionService/tests/IngestionService.unit.test.tsindex 97127c504b..2b59a9512c 100644--- a/worker/src/services/IngestionService/tests/IngestionService.unit.test.ts+++ b/worker/src/services/IngestionService/tests/IngestionService.unit.test.ts@@ -157,6 +157,103 @@ describe("IngestionService unit tests", () => {expect(eventRecord.cost_details).toEqual({ total: 0.03 });});+ it("keeps legacy evaluator metadata for ClickHouse defaults", async () => {+ const ingestionService = new IngestionService(+ {} as any,+ {} as any,+ {} as any,+ {} as any,+ );++ const eventRecord = await ingestionService.createEventRecord(+ {+ projectId: "project-id",+ traceId: "trace-id",+ spanId: "observation-id",+ name: "evaluator",+ type: "SPAN",+ environment: "default",+ startTimeISO: "2026-08-03T00:00:00.000Z",+ endTimeISO: "2026-08-03T00:00:01.000Z",+ metadata: {+ dispatcher_name: "test-dispatcher",+ evaluator_id: "evaluator-1",+ evaluation_rule_id: "rule-1",+ job_configuration_id: "legacy-rule-1",+ evaluator_test: "true",+ },+ source: "otel",+ },+ "otel/project-id/raw-event.json",+ );++ expect(eventRecord.evaluator_id).toBeUndefined();+ expect(eventRecord.evaluation_rule_id).toBeUndefined();+ expect(eventRecord.evaluator_execution_is_test).toBeUndefined();+ expect(eventRecord.metadata_names).toEqual([+ "dispatcher_name",+ "evaluator_id",+ "evaluation_rule_id",+ "job_configuration_id",+ "evaluator_test",+ ]);+ expect(eventRecord.metadata_values).toEqual([+ "test-dispatcher",+ "evaluator-1",+ "rule-1",+ "legacy-rule-1",+ "true",+ ]);+ });++ it("persists evaluation context in dedicated columns without removing metadata", async () => {+ const ingestionService = new IngestionService(+ {} as any,+ {} as any,+ {} as any,+ {} as any,+ );++ const eventRecord = await ingestionService.createEventRecord(+ {+ projectId: "project-id",+ traceId: "trace-id",+ spanId: "observation-id",+ name: "evaluator",+ type: "SPAN",+ environment: "default",+ startTimeISO: "2026-08-03T00:00:00.000Z",+ endTimeISO: "2026-08-03T00:00:01.000Z",+ metadata: {+ evaluator_id: "legacy-evaluator-1",+ job_configuration_id: "legacy-rule-1",+ evaluator_test: "true",+ },+ evaluationContext: {+ evaluatorId: "evaluator-1",+ evaluationRuleId: "rule-1",+ evaluatorExecutionIsTest: false,+ },+ source: "otel",+ },+ "otel/project-id/raw-event.json",+ );++ expect(eventRecord.evaluator_id).toBe("evaluator-1");+ expect(eventRecord.evaluation_rule_id).toBe("rule-1");+ expect(eventRecord.evaluator_execution_is_test).toBe(false);+ expect(eventRecord.metadata_names).toEqual([+ "evaluator_id",+ "job_configuration_id",+ "evaluator_test",+ ]);+ expect(eventRecord.metadata_values).toEqual([+ "legacy-evaluator-1",+ "legacy-rule-1",+ "true",+ ]);+ });+it("preserves non-JSON model parameter strings on direct events", async () => {const ingestionService = new IngestionService({} as any,@@ -495,6 +592,187 @@ describe("IngestionService unit tests", () => {expect(addToQueue).not.toHaveBeenCalled();});+ it("persists evaluator identifiers after the score JSON round-trip", async () => {+ const addToQueue = vi.fn();+ const ingestionService = new IngestionService(+ {} as any,+ {} as any,+ { addToQueue } as any,+ {} as any,+ );+ const timestamp = "2024-10-12T12:13:14.123Z";++ vi.spyOn(ingestionService as any, "getClickhouseRecord").mockResolvedValue(+ null,+ );+ const scoreEvent = JSON.parse(+ JSON.stringify({+ id: "event-id",+ timestamp,+ type: "score-create",+ body: {+ id: "score-id",+ dataType: "NUMERIC",+ name: "quality",+ value: 1,+ source: "EVAL",+ traceId: "trace-id",+ environment: "default",+ metadata: {+ job_execution_id: "job-1",+ evaluator_id: "legacy-evaluator-1",+ evaluation_rule_id: "legacy-rule-1",+ job_configuration_id: "legacy-job-configuration-1",+ evaluator_test: "true",+ },+ evaluatorId: "evaluator-1",+ evaluationRuleId: "rule-1",+ },+ }),+ ) as ScoreEventType;++ await (ingestionService as any).processScoreEventList({+ projectId: "project-id",+ entityId: "score-id",+ createdAtTimestamp: new Date(timestamp),+ scoreEventList: [scoreEvent],+ attribution: {+ ingestionApiKey: "pk-lf-unit-test",+ ingestionSdkName: "langfuse-test",+ ingestionSdkVersion: "0.0.0",+ },+ });++ expect(addToQueue).toHaveBeenCalledWith(+ TableName.Scores,+ expect.objectContaining({+ evaluator_id: "evaluator-1",+ evaluation_rule_id: "rule-1",+ metadata: {+ job_execution_id: "job-1",+ evaluator_id: "legacy-evaluator-1",+ evaluation_rule_id: "legacy-rule-1",+ job_configuration_id: "legacy-job-configuration-1",+ evaluator_test: "true",+ },+ }),+ );+ });++ it("keeps legacy score metadata for ClickHouse defaults", async () => {+ const addToQueue = vi.fn();+ const ingestionService = new IngestionService(+ {} as any,+ {} as any,+ { addToQueue } as any,+ {} as any,+ );+ const timestamp = "2024-10-12T12:13:14.123Z";++ vi.spyOn(ingestionService as any, "getClickhouseRecord").mockResolvedValue(+ null,+ );++ await (ingestionService as any).processScoreEventList({+ projectId: "project-id",+ entityId: "score-id",+ createdAtTimestamp: new Date(timestamp),+ scoreEventList: [+ {+ id: "event-id",+ timestamp,+ type: "score-create",+ body: {+ id: "score-id",+ dataType: "NUMERIC",+ name: "quality",+ value: 1,+ source: "EVAL",+ traceId: "trace-id",+ environment: "default",+ metadata: {+ job_execution_id: "job-1",+ evaluator_id: "evaluator-1",+ job_configuration_id: "legacy-rule-1",+ evaluator_test: "true",+ },+ },+ },+ ] satisfies ScoreEventType[],+ attribution: {+ ingestionApiKey: "pk-lf-unit-test",+ ingestionSdkName: "langfuse-test",+ ingestionSdkVersion: "0.0.0",+ },+ });++ expect(addToQueue).toHaveBeenCalledWith(+ TableName.Scores,+ expect.objectContaining({+ evaluator_id: undefined,+ evaluation_rule_id: undefined,+ metadata: {+ job_execution_id: "job-1",+ evaluator_id: "evaluator-1",+ job_configuration_id: "legacy-rule-1",+ evaluator_test: "true",+ },+ }),+ );+ });++ it("preserves evaluator identifiers when a later score update omits them", async () => {+ const addToQueue = vi.fn();+ const ingestionService = new IngestionService(+ {} as any,+ {} as any,+ { addToQueue } as any,+ {} as any,+ );+ const timestamp = "2024-10-12T12:13:14.123Z";++ vi.spyOn(ingestionService as any, "getClickhouseRecord").mockResolvedValue({+ ...createTraceScore({+ id: "score-id",+ project_id: "project-id",+ trace_id: "trace-id",+ timestamp: new Date(timestamp).getTime(),+ }),+ evaluator_id: "evaluator-1",+ evaluation_rule_id: "rule-1",+ });++ await (ingestionService as any).processScoreEventList({+ projectId: "project-id",+ entityId: "score-id",+ createdAtTimestamp: new Date(timestamp),+ scoreEventList: [+ {+ id: "event-id",+ timestamp,+ type: "score-update",+ body: {+ id: "score-id",+ comment: "updated",+ },+ } as ScoreEventType,+ ],+ attribution: {+ ingestionApiKey: "pk-lf-unit-test",+ ingestionSdkName: "langfuse-test",+ ingestionSdkVersion: "0.0.0",+ },+ });++ expect(addToQueue).toHaveBeenCalledWith(+ TableName.Scores,+ expect.objectContaining({+ evaluator_id: "evaluator-1",+ evaluation_rule_id: "rule-1",+ }),+ );+ });+it("does not silently reject score batches with unexpected record errors", async () => {const addToQueue = vi.fn();const ingestionService = new IngestionService(