Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(evals): add dedicated evaluator execution columns (#16486)

Tobias Wochinger198e9494fe10
diff --git a/packages/shared/clickhouse/migrations/clustered/0047_add_eval_execution_columns.down.sql b/packages/shared/clickhouse/migrations/clustered/0047_add_eval_execution_columns.down.sql
new file mode 100644
index 0000000000..490efbde32
--- /dev/null
+++ b/packages/shared/clickhouse/migrations/clustered/0047_add_eval_execution_columns.down.sql
@@ -0,0 +1,88 @@
+ALTER TABLE events_core_mv ON CLUSTER default MODIFY QUERY
+SELECT
+ project_id,
+ trace_id,
+ span_id,
+ parent_span_id,
+ start_time,
+ end_time,
+ name,
+ type,
+ environment,
+ version,
+ release,
+ trace_name,
+ user_id,
+ session_id,
+ tags,
+ level,
+ status_message,
+ completion_start_time,
+ is_app_root,
+ bookmarked,
+ public,
+ prompt_id,
+ prompt_name,
+ prompt_version,
+ model_id,
+ provided_model_name,
+ model_parameters,
+ provided_usage_details,
+ usage_details,
+ provided_cost_details,
+ cost_details,
+ usage_pricing_tier_id,
+ usage_pricing_tier_name,
+ tool_definitions,
+ tool_calls,
+ tool_call_names,
+ leftUTF8(input, 200) as input,
+ leftUTF8(output, 200) as output,
+ metadata_names,
+ arrayMap(v -> leftUTF8(v, 200), metadata_values) as metadata_values,
+ experiment_id,
+ experiment_name,
+ experiment_metadata_names,
+ experiment_metadata_values,
+ experiment_description,
+ experiment_dataset_id,
+ experiment_item_id,
+ experiment_item_version,
+ experiment_item_expected_output,
+ experiment_item_metadata_names,
+ experiment_item_metadata_values,
+ experiment_item_root_span_id,
+ source,
+ service_name,
+ service_version,
+ scope_name,
+ scope_version,
+ telemetry_sdk_language,
+ telemetry_sdk_name,
+ telemetry_sdk_version,
+ blob_storage_file_path,
+ event_bytes,
+ created_at,
+ updated_at,
+ event_ts,
+ is_deleted,
+ ingestion_api_key,
+ ingestion_sdk_name,
+ ingestion_sdk_version
+FROM events_full
+SETTINGS enable_full_text_index = 1;
+
+ALTER TABLE events_full ON CLUSTER default DROP COLUMN IF EXISTS evaluator_execution_is_test SETTINGS enable_full_text_index = 1, alter_sync = 2;
+ALTER TABLE events_full ON CLUSTER default DROP COLUMN IF EXISTS evaluator_id SETTINGS enable_full_text_index = 1, alter_sync = 2;
+ALTER TABLE events_full ON CLUSTER default DROP COLUMN IF EXISTS evaluation_rule_id SETTINGS enable_full_text_index = 1, alter_sync = 2;
+
+ALTER TABLE events_core ON CLUSTER default DROP INDEX IF EXISTS idx_evaluator_id SETTINGS enable_full_text_index = 1, alter_sync = 2;
+ALTER TABLE events_core ON CLUSTER default DROP INDEX IF EXISTS idx_evaluation_rule_id SETTINGS enable_full_text_index = 1, alter_sync = 2;
+ALTER TABLE events_core ON CLUSTER default DROP COLUMN IF EXISTS evaluator_execution_is_test SETTINGS enable_full_text_index = 1, alter_sync = 2;
+ALTER TABLE events_core ON CLUSTER default DROP COLUMN IF EXISTS evaluator_id SETTINGS enable_full_text_index = 1, alter_sync = 2;
+ALTER TABLE events_core ON CLUSTER default DROP COLUMN IF EXISTS evaluation_rule_id SETTINGS enable_full_text_index = 1, alter_sync = 2;
+
+ALTER TABLE scores ON CLUSTER default DROP INDEX IF EXISTS idx_evaluator_id SETTINGS alter_sync = 2;
+ALTER TABLE scores ON CLUSTER default DROP INDEX IF EXISTS idx_evaluation_rule_id SETTINGS alter_sync = 2;
+ALTER TABLE scores ON CLUSTER default DROP COLUMN IF EXISTS evaluator_id SETTINGS alter_sync = 2;
+ALTER TABLE scores ON CLUSTER default DROP COLUMN IF EXISTS evaluation_rule_id SETTINGS alter_sync = 2;
diff --git a/packages/shared/clickhouse/migrations/clustered/0047_add_eval_execution_columns.up.sql b/packages/shared/clickhouse/migrations/clustered/0047_add_eval_execution_columns.up.sql
new file mode 100644
index 0000000000..87504a51b4
--- /dev/null
+++ b/packages/shared/clickhouse/migrations/clustered/0047_add_eval_execution_columns.up.sql
@@ -0,0 +1,113 @@
+-- Existing parts resolve these columns through DEFAULT expressions. Skip indexes are added without materializing historical parts to keep this migration metadata-only.
+ALTER TABLE events_full ON CLUSTER default
+ ADD COLUMN IF NOT EXISTS evaluator_id String DEFAULT arrayElement(metadata_values, indexOf(metadata_names, 'evaluator_id')) AFTER metadata_values
+ SETTINGS enable_full_text_index = 1, alter_sync = 2;
+ALTER TABLE events_full ON CLUSTER default
+ ADD COLUMN IF NOT EXISTS evaluation_rule_id String DEFAULT if(notEmpty(arrayElement(metadata_values, indexOf(metadata_names, 'evaluation_rule_id'))), arrayElement(metadata_values, indexOf(metadata_names, 'evaluation_rule_id')), arrayElement(metadata_values, indexOf(metadata_names, 'job_configuration_id'))) AFTER evaluator_id
+ SETTINGS enable_full_text_index = 1, alter_sync = 2;
+ALTER TABLE events_full ON CLUSTER default
+ ADD COLUMN IF NOT EXISTS evaluator_execution_is_test Bool DEFAULT has(metadata_names, 'evaluator_test') AND arrayElement(metadata_values, indexOf(metadata_names, 'evaluator_test')) = 'true' AFTER evaluation_rule_id
+ SETTINGS enable_full_text_index = 1, alter_sync = 2;
+ALTER TABLE events_core ON CLUSTER default
+ ADD COLUMN IF NOT EXISTS evaluator_id String DEFAULT arrayElement(metadata_values, indexOf(metadata_names, 'evaluator_id')) AFTER metadata_values
+ SETTINGS enable_full_text_index = 1, alter_sync = 2;
+ALTER TABLE events_core ON CLUSTER default
+ ADD COLUMN IF NOT EXISTS evaluation_rule_id String DEFAULT if(notEmpty(arrayElement(metadata_values, indexOf(metadata_names, 'evaluation_rule_id'))), arrayElement(metadata_values, indexOf(metadata_names, 'evaluation_rule_id')), arrayElement(metadata_values, indexOf(metadata_names, 'job_configuration_id'))) AFTER evaluator_id
+ SETTINGS enable_full_text_index = 1, alter_sync = 2;
+ALTER TABLE events_core ON CLUSTER default
+ ADD COLUMN IF NOT EXISTS evaluator_execution_is_test Bool DEFAULT has(metadata_names, 'evaluator_test') AND arrayElement(metadata_values, indexOf(metadata_names, 'evaluator_test')) = 'true' AFTER evaluation_rule_id
+ SETTINGS enable_full_text_index = 1, alter_sync = 2;
+ALTER TABLE events_core ON CLUSTER default
+ ADD INDEX IF NOT EXISTS idx_evaluator_id evaluator_id TYPE bloom_filter(0.01) GRANULARITY 1
+ SETTINGS enable_full_text_index = 1, alter_sync = 2;
+ALTER TABLE events_core ON CLUSTER default
+ ADD INDEX IF NOT EXISTS idx_evaluation_rule_id evaluation_rule_id TYPE bloom_filter(0.01) GRANULARITY 1
+ SETTINGS enable_full_text_index = 1, alter_sync = 2;
+ALTER TABLE scores ON CLUSTER default
+ ADD COLUMN IF NOT EXISTS evaluator_id String DEFAULT metadata['evaluator_id'] AFTER metadata
+ SETTINGS alter_sync = 2;
+ALTER TABLE scores ON CLUSTER default
+ ADD COLUMN IF NOT EXISTS evaluation_rule_id String DEFAULT if(notEmpty(metadata['evaluation_rule_id']), metadata['evaluation_rule_id'], metadata['job_configuration_id']) AFTER evaluator_id
+ SETTINGS alter_sync = 2;
+ALTER TABLE scores ON CLUSTER default
+ ADD INDEX IF NOT EXISTS idx_evaluator_id evaluator_id TYPE bloom_filter(0.001) GRANULARITY 1
+ SETTINGS alter_sync = 2;
+ALTER TABLE scores ON CLUSTER default
+ ADD INDEX IF NOT EXISTS idx_evaluation_rule_id evaluation_rule_id TYPE bloom_filter(0.001) GRANULARITY 1
+ SETTINGS alter_sync = 2;
+ALTER TABLE events_core_mv ON CLUSTER default MODIFY QUERY
+SELECT
+ project_id,
+ trace_id,
+ span_id,
+ parent_span_id,
+ start_time,
+ end_time,
+ name,
+ type,
+ environment,
+ version,
+ release,
+ trace_name,
+ user_id,
+ session_id,
+ tags,
+ level,
+ status_message,
+ completion_start_time,
+ is_app_root,
+ bookmarked,
+ public,
+ prompt_id,
+ prompt_name,
+ prompt_version,
+ model_id,
+ provided_model_name,
+ model_parameters,
+ provided_usage_details,
+ usage_details,
+ provided_cost_details,
+ cost_details,
+ usage_pricing_tier_id,
+ usage_pricing_tier_name,
+ tool_definitions,
+ tool_calls,
+ tool_call_names,
+ leftUTF8(input, 200) as input,
+ leftUTF8(output, 200) as output,
+ metadata_names,
+ arrayMap(v -> leftUTF8(v, 200), metadata_values) as metadata_values,
+ evaluator_id,
+ evaluation_rule_id,
+ evaluator_execution_is_test,
+ experiment_id,
+ experiment_name,
+ experiment_metadata_names,
+ experiment_metadata_values,
+ experiment_description,
+ experiment_dataset_id,
+ experiment_item_id,
+ experiment_item_version,
+ experiment_item_expected_output,
+ experiment_item_metadata_names,
+ experiment_item_metadata_values,
+ experiment_item_root_span_id,
+ source,
+ service_name,
+ service_version,
+ scope_name,
+ scope_version,
+ telemetry_sdk_language,
+ telemetry_sdk_name,
+ telemetry_sdk_version,
+ blob_storage_file_path,
+ event_bytes,
+ created_at,
+ updated_at,
+ event_ts,
+ is_deleted,
+ ingestion_api_key,
+ ingestion_sdk_name,
+ ingestion_sdk_version
+FROM events_full
+SETTINGS enable_full_text_index = 1;
diff --git a/packages/shared/clickhouse/migrations/unclustered/0047_add_eval_execution_columns.down.sql b/packages/shared/clickhouse/migrations/unclustered/0047_add_eval_execution_columns.down.sql
new file mode 100644
index 0000000000..6f1ed617d1
--- /dev/null
+++ b/packages/shared/clickhouse/migrations/unclustered/0047_add_eval_execution_columns.down.sql
@@ -0,0 +1,88 @@
+ALTER TABLE events_core_mv MODIFY QUERY
+SELECT
+ project_id,
+ trace_id,
+ span_id,
+ parent_span_id,
+ start_time,
+ end_time,
+ name,
+ type,
+ environment,
+ version,
+ release,
+ trace_name,
+ user_id,
+ session_id,
+ tags,
+ level,
+ status_message,
+ completion_start_time,
+ is_app_root,
+ bookmarked,
+ public,
+ prompt_id,
+ prompt_name,
+ prompt_version,
+ model_id,
+ provided_model_name,
+ model_parameters,
+ provided_usage_details,
+ usage_details,
+ provided_cost_details,
+ cost_details,
+ usage_pricing_tier_id,
+ usage_pricing_tier_name,
+ tool_definitions,
+ tool_calls,
+ tool_call_names,
+ leftUTF8(input, 200) as input,
+ leftUTF8(output, 200) as output,
+ metadata_names,
+ arrayMap(v -> leftUTF8(v, 200), metadata_values) as metadata_values,
+ experiment_id,
+ experiment_name,
+ experiment_metadata_names,
+ experiment_metadata_values,
+ experiment_description,
+ experiment_dataset_id,
+ experiment_item_id,
+ experiment_item_version,
+ experiment_item_expected_output,
+ experiment_item_metadata_names,
+ experiment_item_metadata_values,
+ experiment_item_root_span_id,
+ source,
+ service_name,
+ service_version,
+ scope_name,
+ scope_version,
+ telemetry_sdk_language,
+ telemetry_sdk_name,
+ telemetry_sdk_version,
+ blob_storage_file_path,
+ event_bytes,
+ created_at,
+ updated_at,
+ event_ts,
+ is_deleted,
+ ingestion_api_key,
+ ingestion_sdk_name,
+ ingestion_sdk_version
+FROM events_full
+SETTINGS enable_full_text_index = 1;
+
+ALTER TABLE events_full DROP COLUMN IF EXISTS evaluator_execution_is_test SETTINGS enable_full_text_index = 1;
+ALTER TABLE events_full DROP COLUMN IF EXISTS evaluator_id SETTINGS enable_full_text_index = 1;
+ALTER TABLE events_full DROP COLUMN IF EXISTS evaluation_rule_id SETTINGS enable_full_text_index = 1;
+
+ALTER TABLE events_core DROP INDEX IF EXISTS idx_evaluator_id SETTINGS enable_full_text_index = 1;
+ALTER TABLE events_core DROP INDEX IF EXISTS idx_evaluation_rule_id SETTINGS enable_full_text_index = 1;
+ALTER TABLE events_core DROP COLUMN IF EXISTS evaluator_execution_is_test SETTINGS enable_full_text_index = 1;
+ALTER TABLE events_core DROP COLUMN IF EXISTS evaluator_id SETTINGS enable_full_text_index = 1;
+ALTER TABLE events_core DROP COLUMN IF EXISTS evaluation_rule_id SETTINGS enable_full_text_index = 1;
+
+ALTER TABLE scores DROP INDEX IF EXISTS idx_evaluator_id;
+ALTER TABLE scores DROP INDEX IF EXISTS idx_evaluation_rule_id;
+ALTER TABLE scores DROP COLUMN IF EXISTS evaluator_id;
+ALTER TABLE scores DROP COLUMN IF EXISTS evaluation_rule_id;
diff --git a/packages/shared/clickhouse/migrations/unclustered/0047_add_eval_execution_columns.up.sql b/packages/shared/clickhouse/migrations/unclustered/0047_add_eval_execution_columns.up.sql
new file mode 100644
index 0000000000..6a979c92f4
--- /dev/null
+++ b/packages/shared/clickhouse/migrations/unclustered/0047_add_eval_execution_columns.up.sql
@@ -0,0 +1,109 @@
+-- Existing parts resolve these columns through DEFAULT expressions. Skip indexes are added without materializing historical parts to keep this migration metadata-only.
+ALTER TABLE events_full
+ ADD COLUMN IF NOT EXISTS evaluator_id String DEFAULT arrayElement(metadata_values, indexOf(metadata_names, 'evaluator_id')) AFTER metadata_values
+ SETTINGS enable_full_text_index = 1;
+ALTER TABLE events_full
+ ADD COLUMN IF NOT EXISTS evaluation_rule_id String DEFAULT if(notEmpty(arrayElement(metadata_values, indexOf(metadata_names, 'evaluation_rule_id'))), arrayElement(metadata_values, indexOf(metadata_names, 'evaluation_rule_id')), arrayElement(metadata_values, indexOf(metadata_names, 'job_configuration_id'))) AFTER evaluator_id
+ SETTINGS enable_full_text_index = 1;
+ALTER TABLE events_full
+ ADD COLUMN IF NOT EXISTS evaluator_execution_is_test Bool DEFAULT has(metadata_names, 'evaluator_test') AND arrayElement(metadata_values, indexOf(metadata_names, 'evaluator_test')) = 'true' AFTER evaluation_rule_id
+ SETTINGS enable_full_text_index = 1;
+ALTER TABLE events_core
+ ADD COLUMN IF NOT EXISTS evaluator_id String DEFAULT arrayElement(metadata_values, indexOf(metadata_names, 'evaluator_id')) AFTER metadata_values
+ SETTINGS enable_full_text_index = 1;
+ALTER TABLE events_core
+ ADD COLUMN IF NOT EXISTS evaluation_rule_id String DEFAULT if(notEmpty(arrayElement(metadata_values, indexOf(metadata_names, 'evaluation_rule_id'))), arrayElement(metadata_values, indexOf(metadata_names, 'evaluation_rule_id')), arrayElement(metadata_values, indexOf(metadata_names, 'job_configuration_id'))) AFTER evaluator_id
+ SETTINGS enable_full_text_index = 1;
+ALTER TABLE events_core
+ ADD COLUMN IF NOT EXISTS evaluator_execution_is_test Bool DEFAULT has(metadata_names, 'evaluator_test') AND arrayElement(metadata_values, indexOf(metadata_names, 'evaluator_test')) = 'true' AFTER evaluation_rule_id
+ SETTINGS enable_full_text_index = 1;
+ALTER TABLE events_core
+ ADD INDEX IF NOT EXISTS idx_evaluator_id evaluator_id TYPE bloom_filter(0.01) GRANULARITY 1
+ SETTINGS enable_full_text_index = 1;
+ALTER TABLE events_core
+ ADD INDEX IF NOT EXISTS idx_evaluation_rule_id evaluation_rule_id TYPE bloom_filter(0.01) GRANULARITY 1
+ SETTINGS enable_full_text_index = 1;
+ALTER TABLE scores
+ ADD COLUMN IF NOT EXISTS evaluator_id String DEFAULT metadata['evaluator_id'] AFTER metadata;
+ALTER TABLE scores
+ ADD COLUMN IF NOT EXISTS evaluation_rule_id String DEFAULT if(notEmpty(metadata['evaluation_rule_id']), metadata['evaluation_rule_id'], metadata['job_configuration_id']) AFTER evaluator_id;
+ALTER TABLE scores
+ ADD INDEX IF NOT EXISTS idx_evaluator_id evaluator_id TYPE bloom_filter(0.001) GRANULARITY 1;
+ALTER TABLE scores
+ ADD INDEX IF NOT EXISTS idx_evaluation_rule_id evaluation_rule_id TYPE bloom_filter(0.001) GRANULARITY 1;
+ALTER TABLE events_core_mv MODIFY QUERY
+SELECT
+ project_id,
+ trace_id,
+ span_id,
+ parent_span_id,
+ start_time,
+ end_time,
+ name,
+ type,
+ environment,
+ version,
+ release,
+ trace_name,
+ user_id,
+ session_id,
+ tags,
+ level,
+ status_message,
+ completion_start_time,
+ is_app_root,
+ bookmarked,
+ public,
+ prompt_id,
+ prompt_name,
+ prompt_version,
+ model_id,
+ provided_model_name,
+ model_parameters,
+ provided_usage_details,
+ usage_details,
+ provided_cost_details,
+ cost_details,
+ usage_pricing_tier_id,
+ usage_pricing_tier_name,
+ tool_definitions,
+ tool_calls,
+ tool_call_names,
+ leftUTF8(input, 200) as input,
+ leftUTF8(output, 200) as output,
+ metadata_names,
+ arrayMap(v -> leftUTF8(v, 200), metadata_values) as metadata_values,
+ evaluator_id,
+ evaluation_rule_id,
+ evaluator_execution_is_test,
+ experiment_id,
+ experiment_name,
+ experiment_metadata_names,
+ experiment_metadata_values,
+ experiment_description,
+ experiment_dataset_id,
+ experiment_item_id,
+ experiment_item_version,
+ experiment_item_expected_output,
+ experiment_item_metadata_names,
+ experiment_item_metadata_values,
+ experiment_item_root_span_id,
+ source,
+ service_name,
+ service_version,
+ scope_name,
+ scope_version,
+ telemetry_sdk_language,
+ telemetry_sdk_name,
+ telemetry_sdk_version,
+ blob_storage_file_path,
+ event_bytes,
+ created_at,
+ updated_at,
+ event_ts,
+ is_deleted,
+ ingestion_api_key,
+ ingestion_sdk_name,
+ ingestion_sdk_version
+FROM events_full
+SETTINGS enable_full_text_index = 1;
diff --git a/packages/shared/src/eventsTable.ts b/packages/shared/src/eventsTable.ts
index bd3c2e936c..931fefd8a7 100644
--- a/packages/shared/src/eventsTable.ts
+++ b/packages/shared/src/eventsTable.ts
@@ -1,5 +1,4 @@
import { type ColumnDefinition } from "./tableDefinitions";
-import { EvalExecutionMetadataKey } from "./features/evals/evalExecutionMetadata";
export const eventsTableHasParentObservationSql = "e.parent_span_id != ''";
export const eventsTableIsRootObservationSqlForAlias = (alias: string) => {
@@ -372,14 +371,14 @@ const eventsTableColsDefinition = [
name: "Evaluator ID",
id: "evaluatorId",
type: "stringOptions",
- internal: `arrayElement(e.metadata_values, indexOf(e.metadata_names, '${EvalExecutionMetadataKey.EVALUATOR_ID}'))`,
+ internal: "e.evaluator_id",
options: [],
},
{
name: "Rule ID",
id: "ruleId",
type: "stringOptions",
- internal: `if(notEmpty(arrayElement(e.metadata_values, indexOf(e.metadata_names, '${EvalExecutionMetadataKey.EVALUATION_RULE_ID}'))), arrayElement(e.metadata_values, indexOf(e.metadata_names, '${EvalExecutionMetadataKey.EVALUATION_RULE_ID}')), arrayElement(e.metadata_values, indexOf(e.metadata_names, '${EvalExecutionMetadataKey.JOB_CONFIGURATION_ID}')))`,
+ internal: "e.evaluation_rule_id",
options: [],
},
{
diff --git a/packages/shared/src/features/evals/evalExecutionMetadata.ts b/packages/shared/src/features/evals/evalExecutionMetadata.ts
index f6c7823ce6..2d350c1fbb 100644
--- a/packages/shared/src/features/evals/evalExecutionMetadata.ts
+++ b/packages/shared/src/features/evals/evalExecutionMetadata.ts
@@ -13,3 +13,9 @@ export const EvalExecutionMetadataKey = {
export type EvalExecutionMetadataKey =
(typeof EvalExecutionMetadataKey)[keyof typeof EvalExecutionMetadataKey];
+
+export type EvalExecutionContext = {
+ evaluatorId?: string;
+ evaluationRuleId?: string;
+ evaluatorExecutionIsTest: boolean;
+};
diff --git a/packages/shared/src/features/query/dataModel.ts b/packages/shared/src/features/query/dataModel.ts
index e007b7e833..2a20ff147d 100644
--- a/packages/shared/src/features/query/dataModel.ts
+++ b/packages/shared/src/features/query/dataModel.ts
@@ -13,7 +13,6 @@ import {
eventsTableTraceNameSqlForAlias,
} from "../../eventsTable";
import { LISTABLE_SCORE_TYPES } from "../../domain/scores";
-import { EvalExecutionMetadataKey } from "../evals/evalExecutionMetadata";
// The data model defines all available dimensions, measures, and the timeDimension for a given view.
// Make sure to update web/src/features/dashboard/lib/dashboardUiTableToViewMapping.ts if you make changes
@@ -888,11 +887,19 @@ const createScoreSpecificDimensions = (
description: "Name of the score (e.g., accuracy, toxicity).",
},
evaluatorId: {
- sql: `coalesce(nullIf(${tableAlias}.metadata['${EvalExecutionMetadataKey.EVALUATOR_ID}'], ''), ${tableAlias}.metadata['${EvalExecutionMetadataKey.JOB_CONFIGURATION_ID}'])`,
+ sql: `${tableAlias}.evaluator_id`,
alias: "evaluatorId",
type: "string",
+ description: "Identifier of the evaluator that produced the score.",
+ highCardinality: true,
+ uiHidden: true,
+ },
+ ruleId: {
+ sql: `${tableAlias}.evaluation_rule_id`,
+ alias: "ruleId",
+ type: "string",
description:
- "Identifier of the evaluator, falling back to its legacy evaluation rule identifier.",
+ "Identifier of the evaluation rule, including legacy job configurations.",
highCardinality: true,
uiHidden: true,
},
diff --git a/packages/shared/src/features/query/server/queryBuilder.filterValidation.test.ts b/packages/shared/src/features/query/server/queryBuilder.filterValidation.test.ts
index 329baa129a..afadf42dfd 100644
--- a/packages/shared/src/features/query/server/queryBuilder.filterValidation.test.ts
+++ b/packages/shared/src/features/query/server/queryBuilder.filterValidation.test.ts
@@ -290,9 +290,26 @@ describe("queryBuilder filter type validation", () => {
version,
);
- expect(query).toContain(
- "coalesce(nullIf(scores_numeric.metadata['evaluator_id'], ''), scores_numeric.metadata['job_configuration_id'])",
+ expect(query).toContain("scores_numeric.evaluator_id");
+ expect(query).toContain("IN ({stringOptionsFilter");
+ },
+ );
+
+ it.each(["v1", "v2"] as const)(
+ "lowers rule score filters in the %s scores view",
+ async (version) => {
+ const { query } = await buildQueryWithFilter(
+ {
+ column: "ruleId",
+ operator: "any of",
+ value: ["rule-1"],
+ type: "stringOptions",
+ },
+ { view: "scores-numeric" },
+ version,
);
+
+ expect(query).toContain("scores_numeric.evaluation_rule_id");
expect(query).toContain("IN ({stringOptionsFilter");
},
);
diff --git a/packages/shared/src/server/evals/codeEvalExecution.ts b/packages/shared/src/server/evals/codeEvalExecution.ts
index 233016f940..3ab20b8f71 100644
--- a/packages/shared/src/server/evals/codeEvalExecution.ts
+++ b/packages/shared/src/server/evals/codeEvalExecution.ts
@@ -1,4 +1,5 @@
import { stringifyValue } from "../../utils/stringChecks";
+import type { EvalExecutionContext } from "../../features/evals/evalExecutionMetadata";
import {
INTERNAL_TRACE_EVENT_SOURCE,
type InternalTraceEventInput,
@@ -221,6 +222,7 @@ export async function runCodeBasedEvaluationDispatch(params: {
hasExperimentContext?: boolean;
traceName: string;
metadata: Record<string, unknown>;
+ evaluationContext?: EvalExecutionContext;
writeTrace?: InternalTraceWriter;
}): Promise<CodeBasedEvaluationDispatchResult> {
const payload = buildCodeEvalPayload({
@@ -253,6 +255,7 @@ export async function runCodeBasedEvaluationDispatch(params: {
payload,
output: dispatchResult,
metadata: params.metadata,
+ evaluationContext: params.evaluationContext,
sourceCode: params.version.sourceCode,
}),
});
@@ -300,6 +303,7 @@ export async function runCodeBasedEvaluationDispatch(params: {
: {}),
error_retryable: errorDetails.retryable,
},
+ evaluationContext: params.evaluationContext,
sourceCode: params.version.sourceCode,
level: "ERROR",
statusMessage: `Code eval execution failed: ${visibleError.message}`,
@@ -323,6 +327,7 @@ function buildCodeEvalTraceInput(params: {
payload: CodeEvalPayload;
output: unknown;
metadata: Record<string, unknown>;
+ evaluationContext?: EvalExecutionContext;
sourceCode: string;
level?: string;
statusMessage?: string;
@@ -345,6 +350,7 @@ function buildCodeEvalTraceInput(params: {
...params.metadata,
code_eval_source_code: params.sourceCode,
},
+ evaluationContext: params.evaluationContext,
source: INTERNAL_TRACE_EVENT_SOURCE,
};
diff --git a/packages/shared/src/server/evals/evalExecutionMetadata.test.ts b/packages/shared/src/server/evals/evalExecutionMetadata.test.ts
index 4b4db392fc..f00506a838 100644
--- a/packages/shared/src/server/evals/evalExecutionMetadata.test.ts
+++ b/packages/shared/src/server/evals/evalExecutionMetadata.test.ts
@@ -1,26 +1,32 @@
import { describe, expect, it } from "vitest";
-import { buildEvalExecutionMetadata } from "./evalExecutionMetadata";
+import { buildEvalExecutionData } from "./evalExecutionMetadata";
-describe("buildEvalExecutionMetadata", () => {
+describe("buildEvalExecutionData", () => {
it("identifies an evaluator test", () => {
expect(
- buildEvalExecutionMetadata({
+ buildEvalExecutionData({
type: "TEST",
evaluatorId: "evaluator-1",
targetTraceId: "trace-1",
targetObservationId: "observation-1",
}),
).toEqual({
- evaluator_id: "evaluator-1",
- evaluator_test: "true",
- target_trace_id: "trace-1",
- target_observation_id: "observation-1",
+ executionMetadata: {
+ evaluator_id: "evaluator-1",
+ evaluator_test: "true",
+ target_trace_id: "trace-1",
+ target_observation_id: "observation-1",
+ },
+ evaluationContext: {
+ evaluatorId: "evaluator-1",
+ evaluatorExecutionIsTest: true,
+ },
});
});
it("identifies a rule-driven evaluator execution", () => {
expect(
- buildEvalExecutionMetadata({
+ buildEvalExecutionData({
type: "JOB",
jobExecutionId: "job-1",
jobConfigurationId: "rule-1",
@@ -33,20 +39,27 @@ describe("buildEvalExecutionMetadata", () => {
targetDatasetItemId: null,
}),
).toEqual({
- job_execution_id: "job-1",
- evaluation_rule_id: "rule-1",
- job_configuration_id: "rule-1",
- evaluation_rule_assignment_id: "assignment-1",
- evaluator_id: "evaluator-1",
- evaluator_version_id: "version-2",
- target_trace_id: "trace-1",
- target_observation_id: "observation-1",
+ executionMetadata: {
+ job_execution_id: "job-1",
+ job_configuration_id: "rule-1",
+ evaluation_rule_id: "rule-1",
+ evaluation_rule_assignment_id: "assignment-1",
+ evaluator_version_id: "version-2",
+ evaluator_id: "evaluator-1",
+ target_trace_id: "trace-1",
+ target_observation_id: "observation-1",
+ },
+ evaluationContext: {
+ evaluatorId: "evaluator-1",
+ evaluationRuleId: "rule-1",
+ evaluatorExecutionIsTest: false,
+ },
});
});
it("identifies a manual v2 evaluator execution without a rule", () => {
expect(
- buildEvalExecutionMetadata({
+ buildEvalExecutionData({
type: "JOB",
jobExecutionId: "job-1",
jobConfigurationId: "evaluator-1",
@@ -57,10 +70,17 @@ describe("buildEvalExecutionMetadata", () => {
targetDatasetItemId: null,
}),
).toEqual({
- job_execution_id: "job-1",
- job_configuration_id: "evaluator-1",
- evaluator_id: "evaluator-1",
- evaluator_version_id: "version-2",
+ executionMetadata: {
+ job_execution_id: "job-1",
+ job_configuration_id: "evaluator-1",
+ evaluator_version_id: "version-2",
+ evaluator_id: "evaluator-1",
+ },
+ evaluationContext: {
+ evaluatorId: "evaluator-1",
+ evaluationRuleId: "evaluator-1",
+ evaluatorExecutionIsTest: false,
+ },
});
});
});
diff --git a/packages/shared/src/server/evals/evalExecutionMetadata.ts b/packages/shared/src/server/evals/evalExecutionMetadata.ts
index 51bf4655a9..7ccc18c685 100644
--- a/packages/shared/src/server/evals/evalExecutionMetadata.ts
+++ b/packages/shared/src/server/evals/evalExecutionMetadata.ts
@@ -1,6 +1,9 @@
-import { EvalExecutionMetadataKey } from "../../features/evals/evalExecutionMetadata";
+import {
+ EvalExecutionMetadataKey,
+ type EvalExecutionContext,
+} from "../../features/evals/evalExecutionMetadata";
-export function buildEvalExecutionMetadata(
+export function buildEvalExecutionData(
params:
| {
type: "TEST";
@@ -21,7 +24,9 @@ export function buildEvalExecutionMetadata(
targetDatasetItemId: string | null;
},
) {
- const metadata =
+ // Keep old workers and legacy trace/observation sinks compatible during the
+ // v4 rollout. This metadata transport can be removed in v5.
+ const executionMetadata =
params.type === "TEST"
? {
[EvalExecutionMetadataKey.EVALUATOR_ID]: params.evaluatorId,
@@ -46,9 +51,9 @@ export function buildEvalExecutionMetadata(
params.assignmentId,
}
: {}),
- [EvalExecutionMetadataKey.EVALUATOR_ID]: params.evaluatorId,
[EvalExecutionMetadataKey.EVALUATOR_VERSION_ID]:
params.evaluatorVersionId,
+ [EvalExecutionMetadataKey.EVALUATOR_ID]: params.evaluatorId,
[EvalExecutionMetadataKey.TARGET_TRACE_ID]: params.targetTraceId,
[EvalExecutionMetadataKey.TARGET_OBSERVATION_ID]:
params.targetObservationId,
@@ -56,7 +61,23 @@ export function buildEvalExecutionMetadata(
params.targetDatasetItemId,
};
- return Object.fromEntries(
- Object.entries(metadata).filter(([, value]) => value != null),
- ) as Record<string, string>;
+ const evaluationContext: EvalExecutionContext =
+ params.type === "TEST"
+ ? {
+ evaluatorId: params.evaluatorId ?? undefined,
+ evaluatorExecutionIsTest: true,
+ }
+ : {
+ evaluatorId: params.evaluatorId,
+ evaluationRuleId:
+ params.evaluationRuleId ?? params.jobConfigurationId,
+ evaluatorExecutionIsTest: false,
+ };
+
+ return {
+ executionMetadata: Object.fromEntries(
+ Object.entries(executionMetadata).filter(([, value]) => value != null),
+ ) as Record<string, string>,
+ evaluationContext,
+ };
}
diff --git a/packages/shared/src/server/llm/ai-sdk/telemetry.ts b/packages/shared/src/server/llm/ai-sdk/telemetry.ts
index b0d6cfcaa1..3b266de579 100644
--- a/packages/shared/src/server/llm/ai-sdk/telemetry.ts
+++ b/packages/shared/src/server/llm/ai-sdk/telemetry.ts
@@ -25,6 +25,7 @@ import { traceException } from "../../instrumentation";
import { logger } from "../../logger";
import {
AI_FEATURE_OTEL_SDK_NAME,
+ buildEvaluationAttributes,
LangfuseOtelSpanAttributes,
} from "../../otel/attributes";
import { publishInternalOtelSpans } from "../../otel/internalTraceOtelWriter";
@@ -146,6 +147,9 @@ export function createAiSdkTelemetryCapture(params: {
const serializedInput =
rootInput !== undefined ? stringifyValue(rootInput) : undefined;
+ const evaluationAttributes = traceSinkParams.evaluationContext
+ ? buildEvaluationAttributes(traceSinkParams.evaluationContext)
+ : undefined;
const rootSpan: Span = tracer.startSpan(
traceSinkParams.traceName,
@@ -159,6 +163,7 @@ export function createAiSdkTelemetryCapture(params: {
traceSinkParams.userId,
}
: {}),
+ ...(evaluationAttributes ?? {}),
...(traceSinkParams.metadata
? {
[LangfuseOtelSpanAttributes.TRACE_METADATA]: JSON.stringify(
@@ -214,6 +219,7 @@ export function createAiSdkTelemetryCapture(params: {
// Experiment linkage goes on every span so every materialized event
// remains associated with the run item root.
...(experimentAttributes ?? {}),
+ ...(evaluationAttributes ?? {}),
...(promptAttributes ?? {}),
[LangfuseOtelSpanAttributes.TRACE_NAME]: traceSinkParams.traceName,
[LangfuseOtelSpanAttributes.ENVIRONMENT]: traceSinkParams.environment,
diff --git a/packages/shared/src/server/llm/ai-sdk/telemetryIntegration.test.ts b/packages/shared/src/server/llm/ai-sdk/telemetryIntegration.test.ts
index 9667cb937e..6c324bc7a6 100644
--- a/packages/shared/src/server/llm/ai-sdk/telemetryIntegration.test.ts
+++ b/packages/shared/src/server/llm/ai-sdk/telemetryIntegration.test.ts
@@ -50,7 +50,18 @@ const traceSinkParams: TraceSinkParams = {
traceId: VALID_TRACE_ID,
traceName: "Execute evaluator: helpfulness",
environment: "langfuse-llm-judge",
- metadata: { job_configuration_id: "evaluator-1" },
+ metadata: {
+ job_execution_id: "job-1",
+ evaluator_id: "evaluator-1",
+ evaluation_rule_id: "legacy-rule-1",
+ job_configuration_id: "legacy-rule-1",
+ evaluator_test: "true",
+ },
+ evaluationContext: {
+ evaluatorId: "evaluator-1",
+ evaluationRuleId: "legacy-rule-1",
+ evaluatorExecutionIsTest: true,
+ },
};
const modelParams: ModelParams = {
@@ -278,8 +289,44 @@ describe("AI SDK telemetry integration", () => {
model: "gpt-4o",
usageDetails: { input: 3, output: 5 },
environment: "langfuse-llm-judge",
- metadata: { job_configuration_id: "evaluator-1" },
+ metadata: {
+ job_execution_id: "job-1",
+ evaluator_id: "evaluator-1",
+ evaluation_rule_id: "legacy-rule-1",
+ job_configuration_id: "legacy-rule-1",
+ evaluator_test: "true",
+ },
});
+ const generationMetadata = (
+ generation.body as {
+ metadata: { attributes?: Record<string, unknown> };
+ }
+ ).metadata;
+ expect(generationMetadata.attributes).toMatchObject({
+ "gen_ai.operation.name": "chat",
+ "langfuse.evaluator.id": "evaluator-1",
+ "langfuse.evaluation.rule.id": "legacy-rule-1",
+ "langfuse.evaluator.execution.is_test": "true",
+ });
+
+ const eventInputs = processor.processToEvent(resourceSpans);
+ expect(eventInputs).toHaveLength(2);
+ for (const eventInput of eventInputs) {
+ expect(eventInput).toMatchObject({
+ evaluationContext: {
+ evaluatorId: "evaluator-1",
+ evaluationRuleId: "legacy-rule-1",
+ evaluatorExecutionIsTest: true,
+ },
+ metadata: {
+ job_execution_id: "job-1",
+ evaluator_id: "evaluator-1",
+ evaluation_rule_id: "legacy-rule-1",
+ job_configuration_id: "legacy-rule-1",
+ evaluator_test: "true",
+ },
+ });
+ }
});
it("tags experiment run items so the ingestion pipeline can schedule experiment evals", async () => {
diff --git a/packages/shared/src/server/llm/internalTraceEvents.ts b/packages/shared/src/server/llm/internalTraceEvents.ts
index 5b4496c711..325142ab59 100644
--- a/packages/shared/src/server/llm/internalTraceEvents.ts
+++ b/packages/shared/src/server/llm/internalTraceEvents.ts
@@ -5,6 +5,7 @@ import {
asString,
asStringArray,
} from "../../utils/objects";
+import type { EvalExecutionContext } from "../../features/evals/evalExecutionMetadata";
import { stringifyValue } from "../../utils/stringChecks";
import {
ObservationTypeDomain,
@@ -92,6 +93,7 @@ export type InternalTraceEventInput = {
blobStorageFilePath?: string;
eventRaw?: string;
eventBytes?: number;
+ evaluationContext?: EvalExecutionContext;
experimentId?: string;
experimentName?: string;
experimentMetadataNames?: string[];
diff --git a/packages/shared/src/server/llm/types.ts b/packages/shared/src/server/llm/types.ts
index caf1e8af41..be393ccc0b 100644
--- a/packages/shared/src/server/llm/types.ts
+++ b/packages/shared/src/server/llm/types.ts
@@ -10,6 +10,7 @@ import type {
InternalTraceEventInput,
InternalTraceExperimentContext,
} from "./internalTraceEvents";
+import type { EvalExecutionContext } from "../../features/evals/evalExecutionMetadata";
// disable lint as this is exported and used in web/worker
@@ -543,6 +544,7 @@ export type TraceSinkParams = {
userId?: string;
sessionId?: string;
metadata?: Record<string, unknown>;
+ evaluationContext?: EvalExecutionContext;
prompt?: {
name: string;
version: number;
diff --git a/packages/shared/src/server/otel/OtelIngestionProcessor.ts b/packages/shared/src/server/otel/OtelIngestionProcessor.ts
index 5334f756c1..d93dedfbd9 100644
--- a/packages/shared/src/server/otel/OtelIngestionProcessor.ts
+++ b/packages/shared/src/server/otel/OtelIngestionProcessor.ts
@@ -494,6 +494,8 @@ export class OtelIngestionProcessor {
spanAttributes,
span,
);
+ const evaluationFields =
+ this.extractEvaluationFields(spanAttributes);
const spanContext = {
spanId,
@@ -658,6 +660,9 @@ export class OtelIngestionProcessor {
// Experiment fields
...experimentFields,
+ // Evaluator execution fields
+ ...evaluationFields,
+
// Tool calling
toolDefinitions,
toolCalls,
@@ -3499,6 +3504,34 @@ export class OtelIngestionProcessor {
};
}
+ private extractEvaluationFields(attributes: Record<string, unknown>) {
+ const evaluatorId = attributes[LangfuseOtelSpanAttributes.EVALUATOR_ID];
+ const evaluationRuleId =
+ attributes[LangfuseOtelSpanAttributes.EVALUATION_RULE_ID];
+ const evaluatorExecutionIsTest =
+ attributes[LangfuseOtelSpanAttributes.EVALUATOR_EXECUTION_IS_TEST];
+
+ if (
+ evaluatorId == null &&
+ evaluationRuleId == null &&
+ evaluatorExecutionIsTest == null
+ ) {
+ return { evaluationContext: undefined };
+ }
+
+ return {
+ evaluationContext: {
+ evaluatorId: evaluatorId ? String(evaluatorId) : undefined,
+ evaluationRuleId: evaluationRuleId
+ ? String(evaluationRuleId)
+ : undefined,
+ evaluatorExecutionIsTest:
+ evaluatorExecutionIsTest === true ||
+ evaluatorExecutionIsTest === "true",
+ },
+ };
+ }
+
/**
* The item version is a pointer to a dataset item version (`valid_from` timestamp),
* not a free-form label; "v1" or "latest" cannot resolve to one, so we drop them.
diff --git a/packages/shared/src/server/otel/attributes.ts b/packages/shared/src/server/otel/attributes.ts
index 7382b6b50f..21358cab9e 100644
--- a/packages/shared/src/server/otel/attributes.ts
+++ b/packages/shared/src/server/otel/attributes.ts
@@ -1,3 +1,5 @@
+import type { EvalExecutionContext } from "../../features/evals/evalExecutionMetadata";
+
export enum LangfuseOtelSpanAttributes {
// Langfuse-Trace attributes
TRACE_NAME = "langfuse.trace.name",
@@ -51,6 +53,32 @@ export enum LangfuseOtelSpanAttributes {
EXPERIMENT_ITEM_METADATA = "langfuse.experiment.item.metadata",
EXPERIMENT_ITEM_ROOT_OBSERVATION_ID = "langfuse.experiment.item.root_observation_id",
EXPERIMENT_ITEM_EXPECTED_OUTPUT = "langfuse.experiment.item.expected_output",
+
+ // Evaluator execution attributes
+ EVALUATOR_ID = "langfuse.evaluator.id",
+ EVALUATION_RULE_ID = "langfuse.evaluation.rule.id",
+ EVALUATOR_EXECUTION_IS_TEST = "langfuse.evaluator.execution.is_test",
+}
+
+export function buildEvaluationAttributes(
+ evaluationContext: EvalExecutionContext,
+) {
+ return {
+ ...(evaluationContext.evaluatorId
+ ? {
+ [LangfuseOtelSpanAttributes.EVALUATOR_ID]:
+ evaluationContext.evaluatorId,
+ }
+ : {}),
+ ...(evaluationContext.evaluationRuleId
+ ? {
+ [LangfuseOtelSpanAttributes.EVALUATION_RULE_ID]:
+ evaluationContext.evaluationRuleId,
+ }
+ : {}),
+ [LangfuseOtelSpanAttributes.EVALUATOR_EXECUTION_IS_TEST]:
+ evaluationContext.evaluatorExecutionIsTest,
+ };
}
/** SDK name for internally published AI-feature product traces. */
diff --git a/packages/shared/src/server/otel/internalTraceOtelWriter.test.ts b/packages/shared/src/server/otel/internalTraceOtelWriter.test.ts
index b2cc517090..e49ad56f12 100644
--- a/packages/shared/src/server/otel/internalTraceOtelWriter.test.ts
+++ b/packages/shared/src/server/otel/internalTraceOtelWriter.test.ts
@@ -37,7 +37,18 @@ const codeEvalRootInput: InternalOtelSpanInput = {
statusMessage: "Code eval execution failed: boom",
input: '{"item":{}}',
output: '{"error":"boom"}',
- metadata: { dispatcher_name: "test-dispatcher" },
+ metadata: {
+ dispatcher_name: "test-dispatcher",
+ evaluator_id: "evaluator-1",
+ evaluation_rule_id: "legacy-rule-1",
+ job_configuration_id: "legacy-rule-1",
+ evaluator_test: "true",
+ },
+ evaluationContext: {
+ evaluatorId: "evaluator-1",
+ evaluationRuleId: "legacy-rule-1",
+ evaluatorExecutionIsTest: true,
+ },
};
// Convert the published OTLP payload through the REAL OTel ingestion
@@ -92,6 +103,22 @@ describe("writeInternalTraceViaOtelIngestion", () => {
});
expect(root.metadata).toMatchObject({
dispatcher_name: "test-dispatcher",
+ evaluator_id: "evaluator-1",
+ evaluation_rule_id: "legacy-rule-1",
+ job_configuration_id: "legacy-rule-1",
+ evaluator_test: "true",
+ });
+ expect(root).toMatchObject({
+ evaluationContext: {
+ evaluatorId: "evaluator-1",
+ evaluationRuleId: "legacy-rule-1",
+ evaluatorExecutionIsTest: true,
+ },
+ });
+ expect(root.metadata.attributes).toMatchObject({
+ "langfuse.evaluator.id": "evaluator-1",
+ "langfuse.evaluation.rule.id": "legacy-rule-1",
+ "langfuse.evaluator.execution.is_test": "true",
});
});
diff --git a/packages/shared/src/server/otel/internalTraceOtelWriter.ts b/packages/shared/src/server/otel/internalTraceOtelWriter.ts
index 325235cc02..7c5509b90b 100644
--- a/packages/shared/src/server/otel/internalTraceOtelWriter.ts
+++ b/packages/shared/src/server/otel/internalTraceOtelWriter.ts
@@ -18,7 +18,10 @@ import {
import { ObservationTypeDomain } from "../../domain/observations";
import type { InternalTraceEventInput } from "../llm/internalTraceEvents";
import { logger } from "../logger";
-import { LangfuseOtelSpanAttributes } from "./attributes";
+import {
+ buildEvaluationAttributes,
+ LangfuseOtelSpanAttributes,
+} from "./attributes";
import {
OtelIngestionProcessor,
type ResourceSpan,
@@ -69,11 +72,15 @@ export function internalTraceEventToOtelAttributes(
| "public"
| "release"
| "traceName"
+ | "evaluationContext"
>,
options: { isRoot: boolean },
): Attributes {
const observationType = observationTypeToOtelAttribute(eventInput.type);
const metadataJson = asOtelAttributeString(eventInput.metadata);
+ const evaluationAttributes = eventInput.evaluationContext
+ ? buildEvaluationAttributes(eventInput.evaluationContext)
+ : undefined;
const { isRoot } = options;
return {
@@ -101,6 +108,7 @@ export function internalTraceEventToOtelAttributes(
...(metadataJson !== undefined
? { [LangfuseOtelSpanAttributes.OBSERVATION_METADATA]: metadataJson }
: {}),
+ ...(evaluationAttributes ?? {}),
...(eventInput.modelName !== undefined
? { [LangfuseOtelSpanAttributes.OBSERVATION_MODEL]: eventInput.modelName }
: {}),
@@ -271,6 +279,7 @@ export type InternalOtelSpanInput = Pick<
| "input"
| "output"
| "metadata"
+ | "evaluationContext"
>;
/**
diff --git a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.test.ts b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.test.ts
index c95e97a516..7fe98aebe8 100644
--- a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.test.ts
+++ b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.test.ts
@@ -38,6 +38,21 @@ describe("EventsAggregationQueryBuilder", () => {
"(e.parent_span_id = '' OR e.is_app_root = true) AND e.name <> ''",
);
});
+
+ it("promotes evaluator execution fields into the trace aggregation", () => {
+ const { query } = new EventsAggregationQueryBuilder({
+ projectId: "test-project",
+ })
+ .selectFieldSet("all")
+ .buildWithParams();
+
+ expect(query).toContain(
+ "argMaxIf(evaluator_id, event_ts, evaluator_id <> '') AS evaluator_id",
+ );
+ expect(query).toContain(
+ "argMaxIf(evaluation_rule_id, event_ts, evaluation_rule_id <> '') AS evaluation_rule_id",
+ );
+ });
});
describe("EventsSessionAggregationQueryBuilder", () => {
diff --git a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts
index c7c75a955d..94220e145f 100644
--- a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts
+++ b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts
@@ -569,6 +569,12 @@ const EVENTS_AGGREGATION_FIELDS = {
tags: "argMaxIf(tags, event_ts, notEmpty(tags)) AS tags",
release: "argMaxIf(release, event_ts, release <> '') AS release",
+ // Evaluator execution fields are stamped on every internal evaluation span.
+ evaluator_id:
+ "argMaxIf(evaluator_id, event_ts, evaluator_id <> '') AS evaluator_id",
+ evaluation_rule_id:
+ "argMaxIf(evaluation_rule_id, event_ts, evaluation_rule_id <> '') AS evaluation_rule_id",
+
// experiment fields
experiment_id: "any(experiment_id) as experiment_id",
} as const;
diff --git a/packages/shared/src/server/repositories/definitions.ts b/packages/shared/src/server/repositories/definitions.ts
index 552d2c2e5e..819b3ff2fa 100644
--- a/packages/shared/src/server/repositories/definitions.ts
+++ b/packages/shared/src/server/repositories/definitions.ts
@@ -225,6 +225,8 @@ export const scoreRecordBaseSchema = z.object({
source: z.string(),
comment: z.string().nullish(),
metadata: z.record(z.string(), z.string()),
+ evaluator_id: z.string().optional(),
+ evaluation_rule_id: z.string().optional(),
author_user_id: z.string().nullish(),
config_id: z.string().nullish(),
data_type: z.string(),
@@ -517,6 +519,9 @@ export const eventRecordBaseSchema = z.object({
// Metadata
metadata_names: z.array(z.string()).default([]),
metadata_values: z.array(z.string()).default([]),
+ evaluator_id: z.string().optional(),
+ evaluation_rule_id: z.string().optional(),
+ evaluator_execution_is_test: z.boolean().optional(),
// Experiment properties
experiment_id: z.string().nullish(),
diff --git a/packages/shared/src/server/repositories/events.ts b/packages/shared/src/server/repositories/events.ts
index 941321e60a..9763d91f7e 100644
--- a/packages/shared/src/server/repositories/events.ts
+++ b/packages/shared/src/server/repositories/events.ts
@@ -6,10 +6,6 @@ import type {
ObservationType,
} from "../../domain";
import { env } from "../../env";
-import {
- EvalExecutionMetadataKey,
- type EvalExecutionMetadataKey as EvalExecutionMetadataKeyType,
-} from "../../features/evals/evalExecutionMetadata";
import {
InternalServerError,
InvalidRequestError,
@@ -3404,34 +3400,16 @@ export const getTraceMetadataByIdsFromEvents = async (props: {
});
};
-const eventMetadataValue = (key: EvalExecutionMetadataKeyType) =>
- `arrayElement(e.metadata_values, indexOf(e.metadata_names, '${key}'))`;
-
-/**
- * Value of the first key that is present, so callers can read a renamed
- * metadata key while executions written before the rename still resolve.
- */
-const eventMetadataValueWithFallback = (
- keys: readonly EvalExecutionMetadataKeyType[],
-) =>
- keys
- .slice(0, -1)
- .reduceRight(
- (fallback, key) =>
- `if(notEmpty(${eventMetadataValue(key)}), ${eventMetadataValue(key)}, ${fallback})`,
- eventMetadataValue(keys[keys.length - 1]!),
- );
-
-const hasAnyEventMetadataKey = (
- keys: readonly EvalExecutionMetadataKeyType[],
-) => keys.map((key) => `has(e.metadata_names, '${key}')`).join(" OR ");
+const evalExecutionIdentifierColumn = (identifier: "evaluator" | "rule") =>
+ identifier === "evaluator" ? "e.evaluator_id" : "e.evaluation_rule_id";
-const evaluatorTestEventCondition = `(has(e.metadata_names, '${EvalExecutionMetadataKey.EVALUATOR_TEST}') AND ${eventMetadataValue(EvalExecutionMetadataKey.EVALUATOR_TEST)} = 'true') OR startsWith(e.trace_name, 'Test evaluator')`;
+const evaluatorTestEventCondition =
+ "e.evaluator_execution_is_test OR startsWith(e.trace_name, 'Test evaluator')";
-const traceCostsByMetadata = (params: {
+const traceCostsByIdentifier = (params: {
projectId: string;
- metadata: Array<{
- keys: readonly EvalExecutionMetadataKeyType[];
+ identifiers: Array<{
+ identifier: "evaluator" | "rule";
alias: string;
}>;
additionalSelect?: string[];
@@ -3441,10 +3419,10 @@ const traceCostsByMetadata = (params: {
groupByColumn: "e.trace_id",
selectExpression: [
"e.trace_id as trace_id",
- ...params.metadata.map(
- ({ keys, alias }) =>
- `anyIf(${eventMetadataValueWithFallback(keys)}, ${hasAnyEventMetadataKey(keys)}) as ${alias}`,
- ),
+ ...params.identifiers.map(({ identifier, alias }) => {
+ const identifierColumn = evalExecutionIdentifierColumn(identifier);
+ return `anyIf(${identifierColumn}, notEmpty(${identifierColumn})) as ${alias}`;
+ }),
"sum(e.total_cost) as trace_total_cost",
...(params.additionalSelect ?? []),
].join(", "),
@@ -3473,16 +3451,13 @@ const getCostMetricsByMetadataIds = async <
projectId: string;
metadataIds: string[];
fields: TFields;
- metadataKeys: readonly Exclude<
- EvalExecutionMetadataKeyType,
- typeof EvalExecutionMetadataKey.EVALUATOR_TEST
- >[];
+ identifier: "evaluator" | "rule";
}) => {
if (params.metadataIds.length === 0) return [];
- const traceCostsBuilder = traceCostsByMetadata({
+ const traceCostsBuilder = traceCostsByIdentifier({
projectId: params.projectId,
- metadata: [{ keys: params.metadataKeys, alias: "metadata_id" }],
+ identifiers: [{ identifier: params.identifier, alias: "metadata_id" }],
additionalSelect: [
`countIf(${evaluatorTestEventCondition}) as test_event_count`,
],
@@ -3533,7 +3508,7 @@ export const getAvgCostByEvaluatorIds = async (
projectId,
metadataIds: evaluatorIds,
fields: ["avgCost", "executionCount"],
- metadataKeys: [EvalExecutionMetadataKey.EVALUATOR_ID],
+ identifier: "evaluator",
});
return metrics.map(({ metadataId, ...costs }) => ({
evaluatorId: metadataId,
@@ -3549,13 +3524,7 @@ export const getTotalCostByRule = async (
projectId,
metadataIds: ruleIds,
fields: ["totalCost"],
- // Rules replaced job configurations, but executions written before the
- // rename only carry `job_configuration_id`, and the v1 evals UI still
- // reads costs by job-configuration id.
- metadataKeys: [
- EvalExecutionMetadataKey.EVALUATION_RULE_ID,
- EvalExecutionMetadataKey.JOB_CONFIGURATION_ID,
- ],
+ identifier: "rule",
});
return metrics.map(({ metadataId, totalCost }) => ({
ruleId: metadataId,
@@ -3571,7 +3540,7 @@ export const getTotalCostByEvaluatorIds = async (
projectId,
metadataIds: evaluatorIds,
fields: ["totalCost"],
- metadataKeys: [EvalExecutionMetadataKey.EVALUATOR_ID],
+ identifier: "evaluator",
});
return metrics.map(({ metadataId, totalCost }) => ({
evaluatorId: metadataId,
@@ -3579,7 +3548,8 @@ export const getTotalCostByEvaluatorIds = async (
}));
};
-// Temporary compatibility path: remove in a few weeks in favor of evaluator_id.
+// Prompt-experiment executions still use the trace-name fallback until every
+// execution is linked to an evaluator id.
export const getTotalCostByEvaluatorTraceNames = async (
projectId: string,
traceNames: string[],
@@ -3635,12 +3605,12 @@ export const getLatestEvaluatorRunCost = async (
projectId: string,
evaluatorId: string,
) => {
- const builder = traceCostsByMetadata({
+ const builder = traceCostsByIdentifier({
projectId,
- metadata: [
+ identifiers: [
{
- keys: [EvalExecutionMetadataKey.EVALUATOR_ID],
- alias: EvalExecutionMetadataKey.EVALUATOR_ID,
+ identifier: "evaluator",
+ alias: "evaluator_id",
},
],
additionalSelect: [
@@ -3648,10 +3618,7 @@ export const getLatestEvaluatorRunCost = async (
"countIf(e.type = 'GENERATION') as generation_count",
],
})
- .havingRaw(
- `${EvalExecutionMetadataKey.EVALUATOR_ID} = {evaluatorId: String}`,
- { evaluatorId },
- )
+ .havingRaw("evaluator_id = {evaluatorId: String}", { evaluatorId })
.havingRaw("generation_count > 0")
.orderBy("ORDER BY timestamp DESC, trace_id DESC")
.limit(1);
@@ -3718,29 +3685,20 @@ export const getRecentRuleExecutionTraces = async (
) => {
if (ruleIds.length === 0) return [];
- const evaluationRuleId = eventMetadataValueWithFallback([
- EvalExecutionMetadataKey.EVALUATION_RULE_ID,
- EvalExecutionMetadataKey.JOB_CONFIGURATION_ID,
- ]);
- const hasEvaluationRuleId = hasAnyEventMetadataKey([
- EvalExecutionMetadataKey.EVALUATION_RULE_ID,
- EvalExecutionMetadataKey.JOB_CONFIGURATION_ID,
- ]);
const builder = new EventsAggQueryBuilder({
projectId,
- groupByColumn: `e.trace_id, ${EvalExecutionMetadataKey.EVALUATION_RULE_ID}`,
+ groupByColumn: "e.trace_id, e.evaluation_rule_id",
selectExpression: [
"e.trace_id as id",
- `${evaluationRuleId} as ${EvalExecutionMetadataKey.EVALUATION_RULE_ID}`,
+ "e.evaluation_rule_id as evaluation_rule_id",
"multiIf(countIf(e.level = 'ERROR') > 0, 'ERROR', countIf(e.level = 'WARNING') > 0, 'WARNING', 'DEFAULT') as level",
"min(e.start_time) as timestamp",
].join(", "),
})
.whereRaw("e.start_time > now() - INTERVAL 7 DAY")
- .whereRaw(`(${hasEvaluationRuleId})`)
- .whereRaw(`${evaluationRuleId} IN ({ruleIds: Array(String)})`, { ruleIds })
+ .whereRaw("e.evaluation_rule_id IN ({ruleIds: Array(String)})", { ruleIds })
.orderBy("ORDER BY timestamp DESC, id DESC")
- .limitByCount(5, EvalExecutionMetadataKey.EVALUATION_RULE_ID);
+ .limitByCount(5, "evaluation_rule_id");
const { query, params } = builder.buildWithParams();
const rows = await queryClickhouse<
@@ -3748,7 +3706,7 @@ export const getRecentRuleExecutionTraces = async (
id: string;
level: string;
timestamp: string;
- } & Record<typeof EvalExecutionMetadataKey.EVALUATION_RULE_ID, string>
+ } & { evaluation_rule_id: string }
>({
query,
params,
@@ -3758,7 +3716,7 @@ export const getRecentRuleExecutionTraces = async (
return rows.map((row) => ({
id: row.id,
- ruleId: row[EvalExecutionMetadataKey.EVALUATION_RULE_ID],
+ ruleId: row.evaluation_rule_id,
level: row.level,
timestamp: parseClickhouseUTCDateTimeFormat(row.timestamp),
}));
diff --git a/packages/shared/src/server/repositories/scores.ts b/packages/shared/src/server/repositories/scores.ts
index 82c68a56c8..761e28954a 100644
--- a/packages/shared/src/server/repositories/scores.ts
+++ b/packages/shared/src/server/repositories/scores.ts
@@ -225,7 +225,9 @@ const formatMetadataSelect = (
includeHasMetadata: boolean,
) => {
return [
- !excludeMetadata ? "*" : "* EXCEPT (metadata)",
+ !excludeMetadata
+ ? "* EXCEPT (evaluator_id, evaluation_rule_id)"
+ : "* EXCEPT (metadata, evaluator_id, evaluation_rule_id)",
includeHasMetadata
? "length(mapKeys(s.metadata)) > 0 AS has_metadata"
: null,
@@ -644,14 +646,7 @@ export const getScoresForObservations = async <
includeHasMetadata = false,
} = props;
- const select = [
- !excludeMetadata ? "*" : "* EXCEPT (metadata)",
- includeHasMetadata
- ? "length(mapKeys(s.metadata)) > 0 AS has_metadata"
- : null,
- ]
- .filter((s) => s != null)
- .join(", ");
+ const select = formatMetadataSelect(excludeMetadata, includeHasMetadata);
const query = `
select
@@ -1559,7 +1554,7 @@ const getScoresUiGenericFromEvents = async <T>(props: {
s.source,
s.data_type,
s.comment,
- s.metadata['evaluator_id'] AS evaluator_id,
+ s.evaluator_id,
${excludeMetadata ? "" : "s.metadata,"}
s.trace_id,
s.session_id,
diff --git a/packages/shared/src/server/tableMappings/mapEventsTable.ts b/packages/shared/src/server/tableMappings/mapEventsTable.ts
index cee429475b..96faf84172 100644
--- a/packages/shared/src/server/tableMappings/mapEventsTable.ts
+++ b/packages/shared/src/server/tableMappings/mapEventsTable.ts
@@ -2,7 +2,6 @@
// The frontend only sends the column names to the backend. This needs to be changed in the future to send column IDs.
import { UiColumnMappings } from "../../tableDefinitions";
-import { EvalExecutionMetadataKey } from "../../features/evals/evalExecutionMetadata";
import {
eventsTableHasParentObservationSql,
eventsTableIsRootObservationSql,
@@ -220,13 +219,13 @@ export const eventsTableNativeUiColumnDefinitions: UiColumnMappings = [
uiTableName: "Evaluator ID",
uiTableId: "evaluatorId",
clickhouseTableName: "events_proto",
- clickhouseSelect: `arrayElement(e.metadata_values, indexOf(e.metadata_names, '${EvalExecutionMetadataKey.EVALUATOR_ID}'))`,
+ clickhouseSelect: "e.evaluator_id",
},
{
uiTableName: "Rule ID",
uiTableId: "ruleId",
clickhouseTableName: "events_proto",
- clickhouseSelect: `if(notEmpty(arrayElement(e.metadata_values, indexOf(e.metadata_names, '${EvalExecutionMetadataKey.EVALUATION_RULE_ID}'))), arrayElement(e.metadata_values, indexOf(e.metadata_names, '${EvalExecutionMetadataKey.EVALUATION_RULE_ID}')), arrayElement(e.metadata_values, indexOf(e.metadata_names, '${EvalExecutionMetadataKey.JOB_CONFIGURATION_ID}')))`,
+ clickhouseSelect: "e.evaluation_rule_id",
},
{
uiTableName: "Version",
diff --git a/packages/shared/src/server/tableMappings/mapScoresTable.ts b/packages/shared/src/server/tableMappings/mapScoresTable.ts
index cd2ddbb7a6..5b521d43ab 100644
--- a/packages/shared/src/server/tableMappings/mapScoresTable.ts
+++ b/packages/shared/src/server/tableMappings/mapScoresTable.ts
@@ -63,8 +63,15 @@ export const scoresTableUiColumnDefinitions: UiColumnMappings = [
uiTableName: "Evaluator ID",
uiTableId: "evaluatorId",
clickhouseTableName: "scores",
- clickhouseSelect:
- "coalesce(nullIf(s.metadata['evaluator_id'], ''), s.metadata['job_configuration_id'])",
+ clickhouseSelect: "evaluator_id",
+ queryPrefix: "s",
+ },
+ {
+ uiTableName: "Rule ID",
+ uiTableId: "ruleId",
+ clickhouseTableName: "scores",
+ clickhouseSelect: "evaluation_rule_id",
+ queryPrefix: "s",
},
{
uiTableName: "Value",
diff --git a/packages/shared/src/tableDefinitions/scoresTable.ts b/packages/shared/src/tableDefinitions/scoresTable.ts
index 3c1aca903c..91fb5b47f4 100644
--- a/packages/shared/src/tableDefinitions/scoresTable.ts
+++ b/packages/shared/src/tableDefinitions/scoresTable.ts
@@ -70,6 +70,14 @@ export const scoresTableCols: ColumnDefinition[] = [
internal: "",
options: [],
},
+ {
+ name: "Rule ID",
+ id: "ruleId",
+ type: "stringOptions",
+ // ClickHouse-only virtual column; see mapScoresTable.ts.
+ internal: "",
+ options: [],
+ },
{
name: "Metadata",
id: "metadata",
diff --git a/web/src/__tests__/server/scores-trpc.servertest.ts b/web/src/__tests__/server/scores-trpc.servertest.ts
index d26bea4dcd..6451fb9b1f 100644
--- a/web/src/__tests__/server/scores-trpc.servertest.ts
+++ b/web/src/__tests__/server/scores-trpc.servertest.ts
@@ -195,7 +195,7 @@ describe("scores trpc", () => {
);
});
- it("filters evaluator scores by recorded evaluator and legacy rule metadata", async () => {
+ it("filters evaluator scores only by recorded evaluator metadata", async () => {
const [evaluator, otherEvaluator] = await Promise.all(
["Evaluator", "Other evaluator"].map((name) =>
prisma.evaluator.create({
@@ -256,13 +256,13 @@ describe("scores trpc", () => {
]);
expect(new Set(scores.scores.map(({ id }) => id))).toEqual(
- new Set([directScore.id, legacyScore.id]),
+ new Set([directScore.id]),
);
expect(new Set(eventScores.scores.map(({ id }) => id))).toEqual(
- new Set([directScore.id, legacyScore.id]),
+ new Set([directScore.id]),
);
- expect(count.totalCount).toBe(2);
- expect(eventCount.totalCount).toBe(2);
+ expect(count.totalCount).toBe(1);
+ expect(eventCount.totalCount).toBe(1);
});
it("does not match empty boolean representations when filtering boolean values", async () => {
diff --git a/web/src/__tests__/server/unit/evaluator-v2-test-evaluator.servertest.ts b/web/src/__tests__/server/unit/evaluator-v2-test-evaluator.servertest.ts
index c7b1e6177f..b2e171da63 100644
--- a/web/src/__tests__/server/unit/evaluator-v2-test-evaluator.servertest.ts
+++ b/web/src/__tests__/server/unit/evaluator-v2-test-evaluator.servertest.ts
@@ -166,7 +166,12 @@ describe("testEvaluator", () => {
}),
],
trace: expect.objectContaining({
+ evaluationContext: {
+ evaluatorId: "evaluator-1",
+ evaluatorExecutionIsTest: true,
+ },
metadata: expect.objectContaining({
+ evaluator_id: "evaluator-1",
evaluator_test: "true",
target_trace_id: "trace-1",
target_observation_id: "observation-1",
@@ -280,7 +285,12 @@ describe("testEvaluator", () => {
},
{ var: "experimentItemMetadata", value: { source: "test" } },
],
+ evaluationContext: {
+ evaluatorId: "evaluator-1",
+ evaluatorExecutionIsTest: true,
+ },
metadata: expect.objectContaining({
+ evaluator_id: "evaluator-1",
evaluator_test: "true",
target_trace_id: "trace-1",
target_observation_id: "observation-1",
diff --git a/web/src/features/evals/server/codeEvalTestRun.ts b/web/src/features/evals/server/codeEvalTestRun.ts
index 8b66a1d02a..e094161f09 100644
--- a/web/src/features/evals/server/codeEvalTestRun.ts
+++ b/web/src/features/evals/server/codeEvalTestRun.ts
@@ -1,6 +1,6 @@
import {
applyCommentFilters,
- buildEvalExecutionMetadata,
+ buildEvalExecutionData,
createW3CTraceId,
extractObservationVariables,
getEventsStreamForEval,
@@ -166,7 +166,6 @@ export async function runCodeEvalTestForEvaluationRule(params: {
sourceCodeLanguage: version.sourceCodeLanguage,
},
evaluatorMetadata: {
- evaluator_id: evaluator.id,
evaluator_version: version.version,
},
});
@@ -244,7 +243,6 @@ async function runCodeEvalTestForObservation(params: {
evaluator: codeTemplate,
version: codeTemplate,
evaluatorMetadata: {
- evaluator_id: codeTemplate.id,
evaluator_version: codeTemplate.version,
},
});
@@ -288,20 +286,20 @@ async function runCodeEvalTestForObservationWithEvaluator(params: {
});
const executionTraceId = createW3CTraceId();
const traceName = `Test evaluator: ${params.evaluator.name}`;
+ const evaluationData = buildEvalExecutionData({
+ type: "TEST",
+ evaluatorId: params.evaluator.id,
+ targetTraceId: params.observation.trace_id,
+ targetObservationId: params.observation.span_id,
+ });
const executionMetadata = {
dispatcher_name: dispatcher.name,
code_eval_runtime: params.version.sourceCodeLanguage,
...params.evaluatorMetadata,
- ...buildEvalExecutionMetadata({
- type: "TEST",
- evaluatorId: params.evaluator.id,
- targetTraceId: params.observation.trace_id,
- targetObservationId: params.observation.span_id,
- }),
+ ...evaluationData.executionMetadata,
score_name: params.scoreName,
target_object: params.target,
};
-
const dispatchOutcome = await runCodeBasedEvaluationDispatch({
dispatcher,
organizationId: params.orgId,
@@ -314,6 +312,7 @@ async function runCodeEvalTestForObservationWithEvaluator(params: {
hasExperimentContext: Boolean(params.observation.experiment_id),
traceName,
metadata: executionMetadata,
+ evaluationContext: evaluationData.evaluationContext,
writeTrace: writeInternalTraceViaOtelIngestion,
});
diff --git a/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.clienttest.ts b/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.clienttest.ts
index e54ececc06..e11accf48c 100644
--- a/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.clienttest.ts
+++ b/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.clienttest.ts
@@ -11,14 +11,13 @@ import {
} from "./evaluatorScoresUrl";
describe("evaluatorScoresUrl", () => {
- it("filters code evaluator scores by evaluator and assigned rule IDs", () => {
+ it("filters code evaluator scores by evaluator ID", () => {
const url = new URL(
evaluatorScoresUrl(
"project-1",
"evaluator-id",
"Code evaluator",
EvalTemplateTypeEnum.CODE,
- ["rule-1", "rule-2"],
),
"https://langfuse.local",
);
@@ -30,7 +29,7 @@ describe("evaluatorScoresUrl", () => {
column: "evaluatorId",
type: "stringOptions",
operator: "any of",
- value: ["evaluator-id", "rule-1", "rule-2"],
+ value: ["evaluator-id"],
},
{
column: "source",
@@ -48,7 +47,6 @@ describe("evaluatorScoresUrl", () => {
"evaluator-id",
"Correctness",
EvalTemplateTypeEnum.LLM_AS_JUDGE,
- ["rule-1"],
),
"https://langfuse.local",
);
diff --git a/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.ts b/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.ts
index 8116310b46..486b2f2745 100644
--- a/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.ts
+++ b/web/src/features/evals/v2/fns/evaluators/evaluatorScoresUrl.ts
@@ -11,7 +11,6 @@ export function evaluatorScoresUrl(
evaluatorId: string,
evaluatorName: string,
evaluatorType: EvalTemplateType,
- assignedRuleIds: string[],
) {
const evaluatorFilter: FilterState[number] =
evaluatorType === EvalTemplateTypeEnum.CODE
@@ -19,7 +18,7 @@ export function evaluatorScoresUrl(
column: "evaluatorId",
type: "stringOptions",
operator: "any of",
- value: [...new Set([evaluatorId, ...assignedRuleIds])],
+ value: [evaluatorId],
}
: {
column: "name",
diff --git a/web/src/features/evals/v2/pages/EvaluatorsPage.tsx b/web/src/features/evals/v2/pages/EvaluatorsPage.tsx
index 0ee3a9bd2e..f7626a24cb 100644
--- a/web/src/features/evals/v2/pages/EvaluatorsPage.tsx
+++ b/web/src/features/evals/v2/pages/EvaluatorsPage.tsx
@@ -494,7 +494,6 @@ export default function EvaluatorsPage() {
row.original.id,
row.original.name,
row.original.type,
- row.original.assignedRuleIds,
),
)
}
diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorService.ts b/web/src/features/evals/v2/server/evaluators/evaluatorService.ts
index fcfde562a4..d9621bb035 100644
--- a/web/src/features/evals/v2/server/evaluators/evaluatorService.ts
+++ b/web/src/features/evals/v2/server/evaluators/evaluatorService.ts
@@ -259,8 +259,8 @@ export class EvaluatorService {
projectId: params.projectId,
evaluatorIds: params.evaluatorIds,
});
- // Prompt-experiment executions do not always carry evaluator_id metadata,
- // but all evaluator execution paths use this trace-name convention.
+ // Prompt-experiment executions do not always carry an evaluator id, but
+ // all evaluator execution paths use this trace-name convention.
const evaluatorIdsByTraceName = new Map<string, string[]>();
for (const evaluator of evaluators) {
const traceName = `Execute evaluator: ${evaluator.name}`;
diff --git a/web/src/features/evals/v2/server/evaluators/testEvaluator.ts b/web/src/features/evals/v2/server/evaluators/testEvaluator.ts
index 82516483af..a83d722343 100644
--- a/web/src/features/evals/v2/server/evaluators/testEvaluator.ts
+++ b/web/src/features/evals/v2/server/evaluators/testEvaluator.ts
@@ -3,7 +3,7 @@ import {
observationVariableMappingList,
} from "@langfuse/shared";
import {
- buildEvalExecutionMetadata,
+ buildEvalExecutionData,
compileLangfuseMediaMessages,
createW3CTraceId,
DefaultEvalModelService,
@@ -62,7 +62,7 @@ export async function testEvaluator(params: {
observation,
variableMapping,
});
- const metadata = buildEvalExecutionMetadata({
+ const executionData = buildEvalExecutionData({
type: "TEST",
evaluatorId:
params.includeEvaluatorLink === false ? null : params.evaluatorId,
@@ -78,14 +78,14 @@ export async function testEvaluator(params: {
evaluatorId: params.evaluatorId,
definition: params.definition,
variables,
- metadata,
+ ...executionData,
})
: await testLlmEvaluator({
projectId: params.projectId,
evaluatorId: params.evaluatorId,
definition: params.definition,
variables,
- metadata,
+ ...executionData,
});
return { ...result, durationMs: Date.now() - startedAt };
@@ -96,7 +96,10 @@ async function testLlmEvaluator(params: {
evaluatorId: string;
definition: Extract<NormalizedEvaluatorDefinition, { type: "LLM_AS_JUDGE" }>;
variables: ExtractedVariable[];
- metadata: ReturnType<typeof buildEvalExecutionMetadata>;
+ executionMetadata: Record<string, string>;
+ evaluationContext: ReturnType<
+ typeof buildEvalExecutionData
+ >["evaluationContext"];
}) {
const modelConfig = await DefaultEvalModelService.fetchValidModelConfig(
params.projectId,
@@ -149,7 +152,8 @@ async function testLlmEvaluator(params: {
traceId: executionTraceId,
traceName: "Test evaluator",
environment: LangfuseInternalTraceEnvironment.LLMJudge,
- metadata: params.metadata,
+ metadata: params.executionMetadata,
+ evaluationContext: params.evaluationContext,
},
});
estimatedCostUsd = await calculateTestRunCost({
@@ -225,7 +229,10 @@ async function testCodeEvaluator(params: {
evaluatorId: string;
definition: Extract<NormalizedEvaluatorDefinition, { type: "CODE" }>;
variables: ExtractedVariable[];
- metadata: ReturnType<typeof buildEvalExecutionMetadata>;
+ executionMetadata: Record<string, string>;
+ evaluationContext: ReturnType<
+ typeof buildEvalExecutionData
+ >["evaluationContext"];
}) {
const dispatcher = resolveConfiguredCodeEvalDispatcher();
if (!dispatcher) {
@@ -236,7 +243,6 @@ async function testCodeEvaluator(params: {
}
const executionTraceId = createW3CTraceId();
-
return runCodeBasedEvaluationDispatch({
dispatcher,
organizationId: params.orgId,
@@ -247,6 +253,7 @@ async function testCodeEvaluator(params: {
version: params.definition,
extractedVariables: params.variables,
traceName: "Test evaluator",
- metadata: params.metadata,
+ metadata: params.executionMetadata,
+ evaluationContext: params.evaluationContext,
});
}
diff --git a/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.test.ts b/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.test.ts
index c8bc9a772c..a323607060 100644
--- a/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.test.ts
+++ b/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.test.ts
@@ -44,6 +44,23 @@ vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
import { executeCodeBasedEvaluation } from "./executeCodeBasedEvaluation";
+const evaluationContext = {
+ evaluatorId: "evaluator-1",
+ evaluationRuleId: "config-1",
+ evaluatorExecutionIsTest: false,
+};
+
+const executeCodeBasedEvaluationWithContext = (
+ params: Omit<
+ Parameters<typeof executeCodeBasedEvaluation>[0],
+ "evaluationContext"
+ >,
+) =>
+ executeCodeBasedEvaluation({
+ ...params,
+ evaluationContext,
+ });
+
describe("executeCodeBasedEvaluation", () => {
beforeEach(() => {
vi.clearAllMocks();
@@ -63,7 +80,7 @@ describe("executeCodeBasedEvaluation", () => {
],
});
- const result = await executeCodeBasedEvaluation({
+ const result = await executeCodeBasedEvaluationWithContext({
projectId: "project-1",
organizationId: "org-1",
evaluatorId: "evaluator-1",
@@ -177,7 +194,7 @@ describe("executeCodeBasedEvaluation", () => {
],
});
- const result = await executeCodeBasedEvaluation({
+ const result = await executeCodeBasedEvaluationWithContext({
projectId: "project-1",
organizationId: "org-1",
job: {
@@ -228,7 +245,7 @@ describe("executeCodeBasedEvaluation", () => {
scores: [{ name: "score", value: 1, dataType: "BOOLEAN" }],
});
- await executeCodeBasedEvaluation({
+ await executeCodeBasedEvaluationWithContext({
projectId: "project-1",
organizationId: "org-1",
job: {
@@ -276,7 +293,7 @@ describe("executeCodeBasedEvaluation", () => {
scores: [{ name: "score", value: 1, dataType: "BOOLEAN" }],
});
- await executeCodeBasedEvaluation({
+ await executeCodeBasedEvaluationWithContext({
projectId: "project-1",
organizationId: "org-1",
job: {
@@ -320,7 +337,7 @@ describe("executeCodeBasedEvaluation", () => {
scores: [{ name: "score", value: 1, dataType: "BOOLEAN" }],
});
- await executeCodeBasedEvaluation({
+ await executeCodeBasedEvaluationWithContext({
projectId: "project-1",
organizationId: "org-1",
job: {
@@ -372,7 +389,7 @@ describe("executeCodeBasedEvaluation", () => {
scores: [{ name: "score", value: 1, dataType: "BOOLEAN" }],
});
- await executeCodeBasedEvaluation({
+ await executeCodeBasedEvaluationWithContext({
projectId: "project-1",
organizationId: "org-1",
job: {
@@ -427,7 +444,7 @@ describe("executeCodeBasedEvaluation", () => {
mocks.writeInternalTrace.mockRejectedValue(new Error("trace write failed"));
await expect(
- executeCodeBasedEvaluation({
+ executeCodeBasedEvaluationWithContext({
projectId: "project-1",
organizationId: "org-1",
job: {
@@ -464,7 +481,7 @@ describe("executeCodeBasedEvaluation", () => {
});
mocks.dispatcher.dispatch.mockRejectedValue(error);
- const promise = executeCodeBasedEvaluation({
+ const promise = executeCodeBasedEvaluationWithContext({
projectId: "project-1",
organizationId: "org-1",
job: {
@@ -538,7 +555,7 @@ describe("executeCodeBasedEvaluation", () => {
parseDispatchResult({ score: 1 }),
);
- const promise = executeCodeBasedEvaluation({
+ const promise = executeCodeBasedEvaluationWithContext({
projectId: "project-1",
organizationId: "org-1",
job: {
@@ -600,7 +617,7 @@ describe("executeCodeBasedEvaluation", () => {
});
mocks.dispatcher.dispatch.mockRejectedValue(error);
- const promise = executeCodeBasedEvaluation({
+ const promise = executeCodeBasedEvaluationWithContext({
projectId: "project-1",
organizationId: "org-1",
job: {
@@ -701,7 +718,7 @@ describe("executeCodeBasedEvaluation", () => {
);
mocks.dispatcher.dispatch.mockRejectedValue(error);
- const promise = executeCodeBasedEvaluation({
+ const promise = executeCodeBasedEvaluationWithContext({
projectId: "project-1",
organizationId: "org-1",
job: {
diff --git a/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.ts b/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.ts
index d818e8e164..19e392c482 100644
--- a/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.ts
+++ b/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.ts
@@ -1,5 +1,8 @@
import { type JobConfiguration, type JobExecution } from "@prisma/client";
-import { type EvalTemplateCodeBased } from "@langfuse/shared";
+import {
+ type EvalExecutionContext,
+ type EvalTemplateCodeBased,
+} from "@langfuse/shared";
import {
CodeEvalExecutionError,
instrumentAsync,
@@ -24,6 +27,7 @@ export async function executeCodeBasedEvaluation(params: {
extractedVariables: ExtractedVariable[];
hasExperimentContext?: boolean;
executionMetadata: Record<string, string>;
+ evaluationContext: EvalExecutionContext;
// Unused by code-based eval; the shared observation processor passes it.
deps?: EvalExecutionDeps;
}): Promise<EvalExecutionResult> {
@@ -70,6 +74,7 @@ export async function executeCodeBasedEvaluation(params: {
hasExperimentContext: params.hasExperimentContext ?? false,
traceName: `Execute evaluator: ${params.template.name}`,
metadata: executionMetadata,
+ evaluationContext: params.evaluationContext,
// Publish via the OTel ingestion pipeline (like LLM-as-a-judge) so the
// trace reaches the legacy tables too in dual write mode — a direct
// events-table write alone 404s in the trace detail view.
@@ -86,6 +91,7 @@ export async function executeCodeBasedEvaluation(params: {
scores: dispatchOutcome.scores,
executionTraceId,
metadata: executionMetadata,
+ evaluationContext: params.evaluationContext,
};
},
);
diff --git a/worker/src/features/evaluation/evalCompletion.ts b/worker/src/features/evaluation/evalCompletion.ts
index 3a73274944..910e3a2a16 100644
--- a/worker/src/features/evaluation/evalCompletion.ts
+++ b/worker/src/features/evaluation/evalCompletion.ts
@@ -1,4 +1,5 @@
import { JobExecutionStatus } from "@prisma/client";
+import type { EvalExecutionContext } from "@langfuse/shared";
import {
logger,
traceException,
@@ -16,6 +17,7 @@ export type EvalExecutionResult = {
scores: CodeEvalScoreWithName[];
executionTraceId: string;
metadata: Record<string, string>;
+ evaluationContext: EvalExecutionContext;
};
export async function completeEvalExecution({
@@ -43,6 +45,7 @@ export async function completeEvalExecution({
environment,
executionTraceId: result.executionTraceId,
executionMetadata: result.metadata,
+ evaluationContext: result.evaluationContext,
});
const [firstScorePayload] = scoreWritePayloads;
diff --git a/worker/src/features/evaluation/evalExecutionDeps.ts b/worker/src/features/evaluation/evalExecutionDeps.ts
index 89fdc76d77..1f56973181 100644
--- a/worker/src/features/evaluation/evalExecutionDeps.ts
+++ b/worker/src/features/evaluation/evalExecutionDeps.ts
@@ -1,6 +1,7 @@
import { randomUUID } from "crypto";
import { z } from "zod";
import { JobExecutionStatus } from "@prisma/client";
+import type { EvalExecutionContext } from "@langfuse/shared";
import { prisma } from "@langfuse/shared/src/db";
import {
buildEventBucketPrefix,
@@ -63,6 +64,7 @@ interface LLMCallParams {
traceName: string;
environment: string;
metadata: Record<string, unknown>;
+ evaluationContext?: EvalExecutionContext;
};
}
@@ -282,6 +284,7 @@ export function createProductionEvalExecutionDeps(): EvalExecutionDeps {
traceName: params.traceSinkParams.traceName,
environment: params.traceSinkParams.environment,
metadata: params.traceSinkParams.metadata,
+ evaluationContext: params.traceSinkParams.evaluationContext,
eventsWriter: createInternalEventsWriter(),
},
});
diff --git a/worker/src/features/evaluation/evalScoreEvent.ts b/worker/src/features/evaluation/evalScoreEvent.ts
index de072f4799..abc15b9de6 100644
--- a/worker/src/features/evaluation/evalScoreEvent.ts
+++ b/worker/src/features/evaluation/evalScoreEvent.ts
@@ -1,5 +1,5 @@
import { randomUUID } from "crypto";
-import { ScoreSourceEnum } from "@langfuse/shared";
+import { ScoreSourceEnum, type EvalExecutionContext } from "@langfuse/shared";
import {
buildDeterministicEvalScoreIds,
eventTypes,
@@ -10,7 +10,12 @@ import {
export type EvalScoreWritePayload = {
eventId: string;
scoreId: string;
- event: ScoreEventType;
+ event: ScoreEventType & {
+ body: ScoreEventType["body"] & {
+ evaluatorId?: string;
+ evaluationRuleId?: string;
+ };
+ };
};
export function buildEvalScoreWritePayloads(params: {
@@ -21,6 +26,7 @@ export function buildEvalScoreWritePayloads(params: {
environment: string;
executionTraceId: string;
executionMetadata: Record<string, string>;
+ evaluationContext: EvalExecutionContext;
}): EvalScoreWritePayload[] {
const scoreIds = buildDeterministicEvalScoreIds({
scores: params.scores,
@@ -30,7 +36,6 @@ export function buildEvalScoreWritePayloads(params: {
return params.scores.map((score, index) => {
const eventId = randomUUID();
const scoreId = scoreIds[index]!;
-
return {
eventId,
scoreId,
@@ -48,13 +53,15 @@ export function buildEvalScoreWritePayloads(params: {
...(score.metadata ?? {}),
...params.executionMetadata,
},
+ evaluatorId: params.evaluationContext.evaluatorId,
+ evaluationRuleId: params.evaluationContext.evaluationRuleId,
configId: score.configId,
source: ScoreSourceEnum.EVAL,
environment: params.environment,
executionTraceId: params.executionTraceId,
value: score.value,
dataType: score.dataType,
- } as ScoreEventType["body"],
+ } as EvalScoreWritePayload["event"]["body"],
},
};
});
diff --git a/worker/src/features/evaluation/evalService.ts b/worker/src/features/evaluation/evalService.ts
index 812b0e7aed..7f8be1ec27 100644
--- a/worker/src/features/evaluation/evalService.ts
+++ b/worker/src/features/evaluation/evalService.ts
@@ -32,7 +32,7 @@ import {
DatasetRunItemUpsertEventType,
classifyEvaluatorLlmError,
blockEvaluator,
- buildEvalExecutionMetadata,
+ buildEvalExecutionData,
EvaluatorBlockSource,
executeLlmEvaluator,
type CodeEvalScoreWithName,
@@ -65,6 +65,7 @@ import {
type EvalOutputResult,
extractValueFromObject,
validateEvaluatorFiltersForTarget,
+ type EvalExecutionContext,
} from "@langfuse/shared";
import { env } from "../../env";
import { prisma } from "@langfuse/shared/src/db";
@@ -944,6 +945,7 @@ export async function runLLMAsJudgeEvaluation({
template,
extractedVariables,
executionMetadata,
+ evaluationContext,
deps,
evaluatorId,
}: {
@@ -954,6 +956,7 @@ export async function runLLMAsJudgeEvaluation({
template: EvalTemplateLlmAsAJudge;
extractedVariables: ExtractedVariable[];
executionMetadata: Record<string, string>;
+ evaluationContext: EvalExecutionContext;
deps: EvalExecutionDeps;
/**
* Evaluator v2 identity, when the execution came from an evaluation rule.
@@ -1141,9 +1144,8 @@ export async function runLLMAsJudgeEvaluation({
traceId: executionTraceId,
traceName: `Execute evaluator: ${template.name}`,
environment: LangfuseInternalTraceEnvironment.LLMJudge,
- metadata: {
- ...executionMetadata,
- },
+ metadata: executionMetadata,
+ evaluationContext,
},
});
llmSpan.setAttribute("eval.llm.outcome", "success");
@@ -1223,6 +1225,7 @@ export async function runLLMAsJudgeEvaluation({
scores,
executionTraceId,
metadata: executionMetadata,
+ evaluationContext,
};
},
);
@@ -1268,7 +1271,7 @@ function toNormalizedScores(params: {
export async function executeLLMAsJudgeEvaluation(
params: Omit<
Parameters<typeof runLLMAsJudgeEvaluation>[0],
- "deps" | "executionMetadata"
+ "deps" | "executionMetadata" | "evaluationContext"
> & {
environment: string;
deps?: EvalExecutionDeps;
@@ -1278,7 +1281,7 @@ export async function executeLLMAsJudgeEvaluation(
},
): Promise<void> {
const deps = params.deps ?? createProductionEvalExecutionDeps();
- const executionMetadata = buildEvalExecutionMetadata({
+ const executionData = buildEvalExecutionData({
type: "JOB",
jobExecutionId: params.jobExecutionId,
jobConfigurationId: params.job.jobConfigurationId,
@@ -1297,7 +1300,7 @@ export async function executeLLMAsJudgeEvaluation(
const result = await runLLMAsJudgeEvaluation({
...params,
deps,
- executionMetadata,
+ ...executionData,
});
await completeEvalExecution({
diff --git a/worker/src/features/evaluation/evaluationHelpers.test.ts b/worker/src/features/evaluation/evaluationHelpers.test.ts
index ccbece6444..446852bda4 100644
--- a/worker/src/features/evaluation/evaluationHelpers.test.ts
+++ b/worker/src/features/evaluation/evaluationHelpers.test.ts
@@ -11,7 +11,7 @@ import {
validateEvalOutputResult,
} from "@langfuse/shared";
import {
- buildEvalExecutionMetadata,
+ buildEvalExecutionData,
type EvaluatorLlmErrorClassification,
} from "@langfuse/shared/src/server";
import { parseDispatchResult } from "../../../../packages/shared/src/server/evals/codeEvalDispatcherTypes";
@@ -267,7 +267,7 @@ describe("evaluation helpers", () => {
});
});
- describe("buildEvalExecutionMetadata", () => {
+ describe("buildEvalExecutionData", () => {
it("should include all provided fields", () => {
const params = {
type: "JOB" as const,
@@ -278,14 +278,20 @@ describe("evaluation helpers", () => {
targetDatasetItemId: "dataset-def",
};
- const result = buildEvalExecutionMetadata(params);
+ const result = buildEvalExecutionData(params);
expect(result).toEqual({
- job_execution_id: "exec-123",
- job_configuration_id: "config-456",
- target_trace_id: "trace-789",
- target_observation_id: "obs-abc",
- target_dataset_item_id: "dataset-def",
+ executionMetadata: {
+ job_execution_id: "exec-123",
+ job_configuration_id: "config-456",
+ target_trace_id: "trace-789",
+ target_observation_id: "obs-abc",
+ target_dataset_item_id: "dataset-def",
+ },
+ evaluationContext: {
+ evaluationRuleId: "config-456",
+ evaluatorExecutionIsTest: false,
+ },
});
});
@@ -299,15 +305,27 @@ describe("evaluation helpers", () => {
targetDatasetItemId: null,
};
- const result = buildEvalExecutionMetadata(params);
+ const result = buildEvalExecutionData(params);
expect(result).toEqual({
- job_execution_id: "exec-123",
- job_configuration_id: "config-456",
+ executionMetadata: {
+ job_execution_id: "exec-123",
+ job_configuration_id: "config-456",
+ },
+ evaluationContext: {
+ evaluationRuleId: "config-456",
+ evaluatorExecutionIsTest: false,
+ },
});
- expect(Object.keys(result)).not.toContain("target_trace_id");
- expect(Object.keys(result)).not.toContain("target_observation_id");
- expect(Object.keys(result)).not.toContain("target_dataset_item_id");
+ expect(Object.keys(result.executionMetadata)).not.toContain(
+ "target_trace_id",
+ );
+ expect(Object.keys(result.executionMetadata)).not.toContain(
+ "target_observation_id",
+ );
+ expect(Object.keys(result.executionMetadata)).not.toContain(
+ "target_dataset_item_id",
+ );
});
});
@@ -513,6 +531,11 @@ describe("evaluation helpers", () => {
});
describe("buildEvalScoreWritePayloads", () => {
+ const evaluationContext = {
+ evaluationRuleId: "rule-1",
+ evaluatorExecutionIsTest: false,
+ };
+
it("should build stable code eval score IDs when different score names reorder", () => {
const originalPayloads = buildEvalScoreWritePayloads({
scores: [
@@ -533,6 +556,7 @@ describe("evaluation helpers", () => {
environment: "production",
executionTraceId: "exec-trace-789",
executionMetadata: { job_execution_id: "job-1" },
+ evaluationContext,
});
const reorderedPayloads = buildEvalScoreWritePayloads({
scores: [
@@ -553,6 +577,7 @@ describe("evaluation helpers", () => {
environment: "production",
executionTraceId: "exec-trace-789",
executionMetadata: { job_execution_id: "job-1" },
+ evaluationContext,
});
expect(originalPayloads[0].scoreId).toBe(reorderedPayloads[1].scoreId);
@@ -584,6 +609,7 @@ describe("evaluation helpers", () => {
environment: "production",
executionTraceId: "exec-trace-789",
executionMetadata: { job_execution_id: "job-1" },
+ evaluationContext,
});
const scoreIds = result.map((payload) => payload.scoreId);
@@ -639,6 +665,7 @@ describe("evaluation helpers", () => {
environment: "production",
executionTraceId: "exec-trace-789",
executionMetadata: { job_execution_id: "job-1" },
+ evaluationContext,
});
expect(result.map((payload) => payload.scoreId)).toEqual(
@@ -670,6 +697,7 @@ describe("evaluation helpers", () => {
environment: "production",
executionTraceId: "exec-trace-789",
executionMetadata: { job_execution_id: "job-1" },
+ evaluationContext,
});
expect(result).toHaveLength(1);
@@ -706,6 +734,7 @@ describe("evaluation helpers", () => {
environment: "production",
executionTraceId: "exec-trace-789",
executionMetadata: { job_execution_id: "job-1" },
+ evaluationContext,
});
expect(result).toHaveLength(1);
@@ -753,6 +782,7 @@ describe("evaluation helpers", () => {
environment: "production",
executionTraceId: "exec-trace-789",
executionMetadata: { job_execution_id: "job-1" },
+ evaluationContext,
});
expect(result).toHaveLength(2);
@@ -798,6 +828,11 @@ describe("evaluation helpers", () => {
job_execution_id: "job-1",
dispatcher_name: "test-dispatcher",
},
+ evaluationContext: {
+ evaluatorId: "evaluator-1",
+ evaluationRuleId: "legacy-rule-1",
+ evaluatorExecutionIsTest: false,
+ },
});
expect(result[0].event.body.metadata).toEqual({
@@ -806,10 +841,18 @@ describe("evaluation helpers", () => {
job_execution_id: "job-1",
dispatcher_name: "test-dispatcher",
});
+ expect(result[0].event.body).toMatchObject({
+ evaluatorId: "evaluator-1",
+ evaluationRuleId: "legacy-rule-1",
+ });
expect(result[1].event.body.metadata).toEqual({
job_execution_id: "job-1",
dispatcher_name: "test-dispatcher",
});
+ expect(result[1].event.body).toMatchObject({
+ evaluatorId: "evaluator-1",
+ evaluationRuleId: "legacy-rule-1",
+ });
});
});
diff --git a/worker/src/features/evaluation/executeLLMAsJudgeEvaluation.test.ts b/worker/src/features/evaluation/executeLLMAsJudgeEvaluation.test.ts
index e1986b0d9b..2ca85d6a37 100644
--- a/worker/src/features/evaluation/executeLLMAsJudgeEvaluation.test.ts
+++ b/worker/src/features/evaluation/executeLLMAsJudgeEvaluation.test.ts
@@ -525,9 +525,12 @@ describe("executeLLMAsJudgeEvaluation", () => {
traceId: expect.any(String),
traceName: `Execute evaluator: ${mockEvalTemplate.name}`,
environment: "langfuse-llm-as-a-judge",
+ evaluationContext: {
+ evaluationRuleId: mockJobExecution.jobConfigurationId,
+ evaluatorExecutionIsTest: false,
+ },
metadata: expect.objectContaining({
job_execution_id: jobExecutionId,
- job_configuration_id: mockJobExecution.jobConfigurationId,
target_trace_id: mockJobExecution.jobInputTraceId,
}),
}),
@@ -844,9 +847,8 @@ describe("executeLLMAsJudgeEvaluation", () => {
createExecutionParams({ job: fullJob, deps }),
);
- const expectedMetadata = {
+ const expectedTraceMetadata = {
job_execution_id: jobExecutionId,
- job_configuration_id: fullJob.jobConfigurationId,
target_trace_id: fullJob.jobInputTraceId,
target_observation_id: "obs-full",
target_dataset_item_id: "dataset-full",
@@ -855,7 +857,11 @@ describe("executeLLMAsJudgeEvaluation", () => {
expect(callLLM).toHaveBeenCalledWith(
expect.objectContaining({
traceSinkParams: expect.objectContaining({
- metadata: expect.objectContaining(expectedMetadata),
+ evaluationContext: {
+ evaluationRuleId: fullJob.jobConfigurationId,
+ evaluatorExecutionIsTest: false,
+ },
+ metadata: expect.objectContaining(expectedTraceMetadata),
}),
}),
);
@@ -864,7 +870,13 @@ describe("executeLLMAsJudgeEvaluation", () => {
expect.objectContaining({
event: expect.objectContaining({
body: expect.objectContaining({
- metadata: expect.objectContaining(expectedMetadata),
+ evaluationRuleId: fullJob.jobConfigurationId,
+ metadata: expect.objectContaining({
+ job_execution_id: jobExecutionId,
+ target_trace_id: fullJob.jobInputTraceId,
+ target_observation_id: "obs-full",
+ target_dataset_item_id: "dataset-full",
+ }),
}),
}),
}),
@@ -897,12 +909,14 @@ describe("executeLLMAsJudgeEvaluation", () => {
name: mockJobConfiguration.scoreName,
value: 0.88,
comment: "Well structured response",
+ configId: undefined,
source: "EVAL",
environment: "production",
executionTraceId: expect.any(String),
+ evaluatorId: undefined,
+ evaluationRuleId: mockJobExecution.jobConfigurationId,
metadata: expect.objectContaining({
job_execution_id: jobExecutionId,
- job_configuration_id: mockJobExecution.jobConfigurationId,
target_trace_id: mockJobExecution.jobInputTraceId,
}),
dataType: "NUMERIC",
diff --git a/worker/src/features/evaluation/observationEval/__tests__/observationEval.e2e.test.ts b/worker/src/features/evaluation/observationEval/__tests__/observationEval.e2e.test.ts
index 3d03a7df41..0eedf1e10e 100644
--- a/worker/src/features/evaluation/observationEval/__tests__/observationEval.e2e.test.ts
+++ b/worker/src/features/evaluation/observationEval/__tests__/observationEval.e2e.test.ts
@@ -88,6 +88,9 @@ const mockEvalExecutionResult = {
],
executionTraceId: "trace-123",
metadata: {},
+ evaluationContext: {
+ evaluatorExecutionIsTest: false,
+ },
};
describe("Observation Eval E2E Pipeline", () => {
diff --git a/worker/src/features/evaluation/observationEval/__tests__/observationEvalProcessor.test.ts b/worker/src/features/evaluation/observationEval/__tests__/observationEvalProcessor.test.ts
index cf9152925e..5136d8d892 100644
--- a/worker/src/features/evaluation/observationEval/__tests__/observationEvalProcessor.test.ts
+++ b/worker/src/features/evaluation/observationEval/__tests__/observationEvalProcessor.test.ts
@@ -99,6 +99,9 @@ const mockEvalExecutionResult = {
],
executionTraceId: "trace-123",
metadata: {},
+ evaluationContext: {
+ evaluatorExecutionIsTest: false,
+ },
};
describe("processObservationEval", () => {
@@ -323,12 +326,14 @@ describe("processObservationEval", () => {
}),
template: expect.objectContaining({ id: version.id }),
executionMetadata: expect.objectContaining({
- evaluation_rule_id: rule.id,
- job_configuration_id: rule.id,
evaluation_rule_assignment_id: assignment.id,
- evaluator_id: evaluator.id,
evaluator_version_id: version.id,
}),
+ evaluationContext: {
+ evaluatorId: evaluator.id,
+ evaluationRuleId: rule.id,
+ evaluatorExecutionIsTest: false,
+ },
// Pausing targets the evaluator, not the rule it ran for.
evaluatorId: evaluator.id,
}),
@@ -492,9 +497,13 @@ describe("processObservationEval", () => {
expect.objectContaining({
config: expect.objectContaining({ id: evaluator.id }),
executionMetadata: expect.objectContaining({
- evaluator_id: evaluator.id,
evaluator_version_id: version.id,
}),
+ evaluationContext: {
+ evaluatorId: evaluator.id,
+ evaluationRuleId: evaluator.id,
+ evaluatorExecutionIsTest: false,
+ },
}),
);
});
@@ -569,10 +578,11 @@ describe("processObservationEval", () => {
expect.objectContaining({
evaluatorId: evaluator.id,
config: expect.objectContaining({ id: legacyConfig.id }),
- executionMetadata: expect.objectContaining({
- evaluation_rule_id: legacyConfig.id,
- evaluator_id: evaluator.id,
- }),
+ evaluationContext: {
+ evaluatorId: evaluator.id,
+ evaluationRuleId: legacyConfig.id,
+ evaluatorExecutionIsTest: false,
+ },
}),
);
});
diff --git a/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts b/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts
index 2fe0646144..81908385b0 100644
--- a/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts
+++ b/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts
@@ -24,7 +24,7 @@ import {
} from "@prisma/client";
import { prisma, JobExecutionStatus } from "@langfuse/shared/src/db";
import { UnrecoverableError } from "../../../errors/UnrecoverableError";
-import { buildEvalExecutionMetadata } from "@langfuse/shared/src/server";
+import { buildEvalExecutionData } from "@langfuse/shared/src/server";
import {
completeEvalExecution,
type EvalExecutionResult,
@@ -233,7 +233,7 @@ export async function processObservationEval(
extractedVariables,
hasExperimentContext: Boolean(observationData.experiment_id),
environment: observationData.environment ?? DEFAULT_TRACE_ENVIRONMENT,
- executionMetadata: buildEvalExecutionMetadata({
+ ...buildEvalExecutionData({
type: "JOB",
jobExecutionId: event.jobExecutionId,
jobConfigurationId: job.jobConfigurationId,
diff --git a/worker/src/services/IngestionService/index.ts b/worker/src/services/IngestionService/index.ts
index 7f077af253..7900f8dab2 100644
--- a/worker/src/services/IngestionService/index.ts
+++ b/worker/src/services/IngestionService/index.ts
@@ -304,6 +304,7 @@ export class IngestionService {
const modelParameters = parseEventModelParameters(
eventData.modelParameters,
);
+ const metadata = eventData.metadata ?? {};
// Runs outside the modelName gate below so model-less events with provided
// usage are still checked.
@@ -338,7 +339,7 @@ export class IngestionService {
projectId: eventData.projectId,
pricingMatchAttributeValues: {
modelParameters,
- metadata: eventData.metadata,
+ metadata,
},
observationRecord: {
id: eventData.spanId,
@@ -357,14 +358,11 @@ export class IngestionService {
const now = this.getMicrosecondTimestamp();
// Flatten raw metadata first (before stringification destroys nested structure)
- const flattened = eventData.metadata
- ? flattenJsonToPathArrays(eventData.metadata)
- : { names: [], values: [] };
+ const flattened = flattenJsonToPathArrays(metadata);
const metadataNames = flattened.names;
// Defensive: coerce null/undefined to empty string for Array(String) ClickHouse column.
// Should not be required as convertValueToPlainJavascript() never returns null.
const metadataValues = flattened.values.map((v) => v ?? "");
-
const eventRecord: EventRecordInsertType = {
// Required identifiers
id: eventData.spanId,
@@ -440,6 +438,10 @@ export class IngestionService {
// Metadata
metadata_names: metadataNames,
metadata_values: metadataValues,
+ evaluator_id: eventData.evaluationContext?.evaluatorId,
+ evaluation_rule_id: eventData.evaluationContext?.evaluationRuleId,
+ evaluator_execution_is_test:
+ eventData.evaluationContext?.evaluatorExecutionIsTest,
// Source/instrumentation metadata
source: eventData.source,
@@ -652,7 +654,14 @@ export class IngestionService {
scoreId: entityId,
projectId,
});
-
+ const rawMetadata = scoreEvent.body.metadata
+ ? convertJsonSchemaToRecord(scoreEvent.body.metadata)
+ : {};
+ const evaluationFields =
+ scoreEvent.body as ScoreEventType["body"] & {
+ evaluatorId?: string;
+ evaluationRuleId?: string;
+ };
return {
id: entityId,
project_id: projectId,
@@ -668,9 +677,9 @@ export class IngestionService {
observation_id: validatedScore.observationId,
config_id: validatedScore.configId,
comment: validatedScore.comment,
- metadata: scoreEvent.body.metadata
- ? convertJsonSchemaToRecord(scoreEvent.body.metadata)
- : {},
+ metadata: convertRecordValuesToString(rawMetadata),
+ evaluator_id: evaluationFields.evaluatorId,
+ evaluation_rule_id: evaluationFields.evaluationRuleId,
string_value: validatedScore.stringValue,
long_string_value: validatedScore.longStringValue,
execution_trace_id: validatedScore.executionTraceId,
diff --git a/worker/src/services/IngestionService/tests/IngestionService.unit.test.ts b/worker/src/services/IngestionService/tests/IngestionService.unit.test.ts
index 97127c504b..2b59a9512c 100644
--- a/worker/src/services/IngestionService/tests/IngestionService.unit.test.ts
+++ b/worker/src/services/IngestionService/tests/IngestionService.unit.test.ts
@@ -157,6 +157,103 @@ describe("IngestionService unit tests", () => {
expect(eventRecord.cost_details).toEqual({ total: 0.03 });
});
+ it("keeps legacy evaluator metadata for ClickHouse defaults", async () => {
+ const ingestionService = new IngestionService(
+ {} as any,
+ {} as any,
+ {} as any,
+ {} as any,
+ );
+
+ const eventRecord = await ingestionService.createEventRecord(
+ {
+ projectId: "project-id",
+ traceId: "trace-id",
+ spanId: "observation-id",
+ name: "evaluator",
+ type: "SPAN",
+ environment: "default",
+ startTimeISO: "2026-08-03T00:00:00.000Z",
+ endTimeISO: "2026-08-03T00:00:01.000Z",
+ metadata: {
+ dispatcher_name: "test-dispatcher",
+ evaluator_id: "evaluator-1",
+ evaluation_rule_id: "rule-1",
+ job_configuration_id: "legacy-rule-1",
+ evaluator_test: "true",
+ },
+ source: "otel",
+ },
+ "otel/project-id/raw-event.json",
+ );
+
+ expect(eventRecord.evaluator_id).toBeUndefined();
+ expect(eventRecord.evaluation_rule_id).toBeUndefined();
+ expect(eventRecord.evaluator_execution_is_test).toBeUndefined();
+ expect(eventRecord.metadata_names).toEqual([
+ "dispatcher_name",
+ "evaluator_id",
+ "evaluation_rule_id",
+ "job_configuration_id",
+ "evaluator_test",
+ ]);
+ expect(eventRecord.metadata_values).toEqual([
+ "test-dispatcher",
+ "evaluator-1",
+ "rule-1",
+ "legacy-rule-1",
+ "true",
+ ]);
+ });
+
+ it("persists evaluation context in dedicated columns without removing metadata", async () => {
+ const ingestionService = new IngestionService(
+ {} as any,
+ {} as any,
+ {} as any,
+ {} as any,
+ );
+
+ const eventRecord = await ingestionService.createEventRecord(
+ {
+ projectId: "project-id",
+ traceId: "trace-id",
+ spanId: "observation-id",
+ name: "evaluator",
+ type: "SPAN",
+ environment: "default",
+ startTimeISO: "2026-08-03T00:00:00.000Z",
+ endTimeISO: "2026-08-03T00:00:01.000Z",
+ metadata: {
+ evaluator_id: "legacy-evaluator-1",
+ job_configuration_id: "legacy-rule-1",
+ evaluator_test: "true",
+ },
+ evaluationContext: {
+ evaluatorId: "evaluator-1",
+ evaluationRuleId: "rule-1",
+ evaluatorExecutionIsTest: false,
+ },
+ source: "otel",
+ },
+ "otel/project-id/raw-event.json",
+ );
+
+ expect(eventRecord.evaluator_id).toBe("evaluator-1");
+ expect(eventRecord.evaluation_rule_id).toBe("rule-1");
+ expect(eventRecord.evaluator_execution_is_test).toBe(false);
+ expect(eventRecord.metadata_names).toEqual([
+ "evaluator_id",
+ "job_configuration_id",
+ "evaluator_test",
+ ]);
+ expect(eventRecord.metadata_values).toEqual([
+ "legacy-evaluator-1",
+ "legacy-rule-1",
+ "true",
+ ]);
+ });
+
it("preserves non-JSON model parameter strings on direct events", async () => {
const ingestionService = new IngestionService(
{} as any,
@@ -495,6 +592,187 @@ describe("IngestionService unit tests", () => {
expect(addToQueue).not.toHaveBeenCalled();
});
+ it("persists evaluator identifiers after the score JSON round-trip", async () => {
+ const addToQueue = vi.fn();
+ const ingestionService = new IngestionService(
+ {} as any,
+ {} as any,
+ { addToQueue } as any,
+ {} as any,
+ );
+ const timestamp = "2024-10-12T12:13:14.123Z";
+
+ vi.spyOn(ingestionService as any, "getClickhouseRecord").mockResolvedValue(
+ null,
+ );
+ const scoreEvent = JSON.parse(
+ JSON.stringify({
+ id: "event-id",
+ timestamp,
+ type: "score-create",
+ body: {
+ id: "score-id",
+ dataType: "NUMERIC",
+ name: "quality",
+ value: 1,
+ source: "EVAL",
+ traceId: "trace-id",
+ environment: "default",
+ metadata: {
+ job_execution_id: "job-1",
+ evaluator_id: "legacy-evaluator-1",
+ evaluation_rule_id: "legacy-rule-1",
+ job_configuration_id: "legacy-job-configuration-1",
+ evaluator_test: "true",
+ },
+ evaluatorId: "evaluator-1",
+ evaluationRuleId: "rule-1",
+ },
+ }),
+ ) as ScoreEventType;
+
+ await (ingestionService as any).processScoreEventList({
+ projectId: "project-id",
+ entityId: "score-id",
+ createdAtTimestamp: new Date(timestamp),
+ scoreEventList: [scoreEvent],
+ attribution: {
+ ingestionApiKey: "pk-lf-unit-test",
+ ingestionSdkName: "langfuse-test",
+ ingestionSdkVersion: "0.0.0",
+ },
+ });
+
+ expect(addToQueue).toHaveBeenCalledWith(
+ TableName.Scores,
+ expect.objectContaining({
+ evaluator_id: "evaluator-1",
+ evaluation_rule_id: "rule-1",
+ metadata: {
+ job_execution_id: "job-1",
+ evaluator_id: "legacy-evaluator-1",
+ evaluation_rule_id: "legacy-rule-1",
+ job_configuration_id: "legacy-job-configuration-1",
+ evaluator_test: "true",
+ },
+ }),
+ );
+ });
+
+ it("keeps legacy score metadata for ClickHouse defaults", async () => {
+ const addToQueue = vi.fn();
+ const ingestionService = new IngestionService(
+ {} as any,
+ {} as any,
+ { addToQueue } as any,
+ {} as any,
+ );
+ const timestamp = "2024-10-12T12:13:14.123Z";
+
+ vi.spyOn(ingestionService as any, "getClickhouseRecord").mockResolvedValue(
+ null,
+ );
+
+ await (ingestionService as any).processScoreEventList({
+ projectId: "project-id",
+ entityId: "score-id",
+ createdAtTimestamp: new Date(timestamp),
+ scoreEventList: [
+ {
+ id: "event-id",
+ timestamp,
+ type: "score-create",
+ body: {
+ id: "score-id",
+ dataType: "NUMERIC",
+ name: "quality",
+ value: 1,
+ source: "EVAL",
+ traceId: "trace-id",
+ environment: "default",
+ metadata: {
+ job_execution_id: "job-1",
+ evaluator_id: "evaluator-1",
+ job_configuration_id: "legacy-rule-1",
+ evaluator_test: "true",
+ },
+ },
+ },
+ ] satisfies ScoreEventType[],
+ attribution: {
+ ingestionApiKey: "pk-lf-unit-test",
+ ingestionSdkName: "langfuse-test",
+ ingestionSdkVersion: "0.0.0",
+ },
+ });
+
+ expect(addToQueue).toHaveBeenCalledWith(
+ TableName.Scores,
+ expect.objectContaining({
+ evaluator_id: undefined,
+ evaluation_rule_id: undefined,
+ metadata: {
+ job_execution_id: "job-1",
+ evaluator_id: "evaluator-1",
+ job_configuration_id: "legacy-rule-1",
+ evaluator_test: "true",
+ },
+ }),
+ );
+ });
+
+ it("preserves evaluator identifiers when a later score update omits them", async () => {
+ const addToQueue = vi.fn();
+ const ingestionService = new IngestionService(
+ {} as any,
+ {} as any,
+ { addToQueue } as any,
+ {} as any,
+ );
+ const timestamp = "2024-10-12T12:13:14.123Z";
+
+ vi.spyOn(ingestionService as any, "getClickhouseRecord").mockResolvedValue({
+ ...createTraceScore({
+ id: "score-id",
+ project_id: "project-id",
+ trace_id: "trace-id",
+ timestamp: new Date(timestamp).getTime(),
+ }),
+ evaluator_id: "evaluator-1",
+ evaluation_rule_id: "rule-1",
+ });
+
+ await (ingestionService as any).processScoreEventList({
+ projectId: "project-id",
+ entityId: "score-id",
+ createdAtTimestamp: new Date(timestamp),
+ scoreEventList: [
+ {
+ id: "event-id",
+ timestamp,
+ type: "score-update",
+ body: {
+ id: "score-id",
+ comment: "updated",
+ },
+ } as ScoreEventType,
+ ],
+ attribution: {
+ ingestionApiKey: "pk-lf-unit-test",
+ ingestionSdkName: "langfuse-test",
+ ingestionSdkVersion: "0.0.0",
+ },
+ });
+
+ expect(addToQueue).toHaveBeenCalledWith(
+ TableName.Scores,
+ expect.objectContaining({
+ evaluator_id: "evaluator-1",
+ evaluation_rule_id: "rule-1",
+ }),
+ );
+ });
+
it("does not silently reject score batches with unexpected record errors", async () => {
const addToQueue = vi.fn();
const ingestionService = new IngestionService(