Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(evals): add payload-backed evaluation rule filters (#16484)

Tobias Wochingerc2931241cb4d
diff --git a/packages/shared/src/features/evals/observationForEval.test.ts b/packages/shared/src/features/evals/observationForEval.test.ts
index bf9dd78aa8..62d1142b1a 100644
--- a/packages/shared/src/features/evals/observationForEval.test.ts
+++ b/packages/shared/src/features/evals/observationForEval.test.ts
@@ -83,6 +83,36 @@ describe("zipObservationToolCalls", () => {
});
describe("mapEventEvalFilterColumnIdToField", () => {
+ it("maps filters backed directly by the evaluation payload", () => {
+ const observation = {
+ provided_model_name: "gpt-4o",
+ prompt_name: "support-agent",
+ prompt_version: 2,
+ release: "2026-08",
+ status_message: "rate limit exceeded",
+ experiment_name: "checkout-eval",
+ } as ObservationForEval;
+
+ expect(
+ mapEventEvalFilterColumnIdToField(observation, "providedModelName"),
+ ).toBe("gpt-4o");
+ expect(mapEventEvalFilterColumnIdToField(observation, "promptName")).toBe(
+ "support-agent",
+ );
+ expect(
+ mapEventEvalFilterColumnIdToField(observation, "promptVersion"),
+ ).toBe(2);
+ expect(mapEventEvalFilterColumnIdToField(observation, "release")).toBe(
+ "2026-08",
+ );
+ expect(
+ mapEventEvalFilterColumnIdToField(observation, "statusMessage"),
+ ).toBe("rate limit exceeded");
+ expect(
+ mapEventEvalFilterColumnIdToField(observation, "experimentName"),
+ ).toBe("checkout-eval");
+ });
+
it("maps the experiment root filter to a boolean", () => {
expect(
mapEventEvalFilterColumnIdToField(
diff --git a/packages/shared/src/features/evals/observationForEval.ts b/packages/shared/src/features/evals/observationForEval.ts
index 9be52be8b1..726753e6e7 100644
--- a/packages/shared/src/features/evals/observationForEval.ts
+++ b/packages/shared/src/features/evals/observationForEval.ts
@@ -175,12 +175,18 @@ export type ObservationEvalFilterColumnInternal =
| "name"
| "environment"
| "level"
+ | "status_message"
| "version"
+ | "release"
| "trace_name"
| "user_id"
| "session_id"
+ | "provided_model_name"
+ | "prompt_name"
+ | "prompt_version"
| "tags"
| "experiment_id"
+ | "experiment_name"
| "experiment_dataset_id"
| "experiment_item_root_span_id"
| "metadata"
@@ -347,6 +353,41 @@ export const observationEvalFilterColumns: ObservationEvalColumnDef[] = [
internal: "version",
nullable: true,
},
+ {
+ name: "Release",
+ id: "release",
+ type: "string",
+ internal: "release",
+ nullable: true,
+ },
+ {
+ name: "Status Message",
+ id: "statusMessage",
+ type: "string",
+ internal: "status_message",
+ nullable: true,
+ },
+ {
+ name: "Provided Model Name",
+ id: "providedModelName",
+ type: "string",
+ internal: "provided_model_name",
+ nullable: true,
+ },
+ {
+ name: "Prompt Name",
+ id: "promptName",
+ type: "string",
+ internal: "prompt_name",
+ nullable: true,
+ },
+ {
+ name: "Prompt Version",
+ id: "promptVersion",
+ type: "number",
+ internal: "prompt_version",
+ nullable: true,
+ },
{
name: "Trace Name",
id: "traceName",
@@ -403,6 +444,13 @@ export const observationEvalFilterColumns: ObservationEvalColumnDef[] = [
options: [], // to be filled at runtime
nullable: true,
},
+ {
+ name: "Experiment Name",
+ id: "experimentName",
+ type: "string",
+ internal: "experiment_name",
+ nullable: true,
+ },
{
name: "Is Experiment Item Root Span",
id: "isExperimentItemRootSpan",
diff --git a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts
index 5d04e00b55..b9fedc4517 100644
--- a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts
+++ b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts
@@ -474,6 +474,7 @@ const FIELD_SETS = {
"toolCalls",
"toolCallNames",
"experimentId",
+ "experimentName",
"experimentItemRootSpanId",
"experimentItemExpectedOutput",
"experimentItemMetadata",
diff --git a/packages/shared/src/server/queries/clickhouse-sql/events-stream-query.test.ts b/packages/shared/src/server/queries/clickhouse-sql/events-stream-query.test.ts
index 279b4aa92e..c28417a7c8 100644
--- a/packages/shared/src/server/queries/clickhouse-sql/events-stream-query.test.ts
+++ b/packages/shared/src/server/queries/clickhouse-sql/events-stream-query.test.ts
@@ -68,6 +68,7 @@ describe("buildEventsStreamQuery", () => {
expect(query).toContain("e.project_id = {projectId: String}");
expect(query).toContain("e.is_deleted = 0");
expect(query).toContain('e.is_app_root as "is_app_root"');
+ expect(query).toContain('e.experiment_name as "experiment_name"');
const orderIndex = query.lastIndexOf("ORDER BY ");
const deduplicationIndex = query.lastIndexOf("LIMIT 1 BY ");
diff --git a/packages/shared/src/server/repositories/events-stream.ts b/packages/shared/src/server/repositories/events-stream.ts
index 0e59db68de..c97fa15ab3 100644
--- a/packages/shared/src/server/repositories/events-stream.ts
+++ b/packages/shared/src/server/repositories/events-stream.ts
@@ -73,6 +73,7 @@ export const getEventsStreamForEval = async (props: {
output: unknown;
metadata: Record<string, unknown> | null;
experiment_id: string | null;
+ experiment_name: string | null;
experiment_item_root_span_id: string | null;
experiment_item_expected_output: string | null;
experiment_item_metadata: Record<string, unknown> | null;
diff --git a/web/src/features/evals/v2/constants/ruleSearchRegistry.ts b/web/src/features/evals/v2/constants/ruleSearchRegistry.ts
index 09d43cb849..012c9e5721 100644
--- a/web/src/features/evals/v2/constants/ruleSearchRegistry.ts
+++ b/web/src/features/evals/v2/constants/ruleSearchRegistry.ts
@@ -18,6 +18,16 @@ export const RULE_FIELD_REGISTRY = fieldRegistryFromColumns(
{ observedOptionsKey: "environment", promptLabel: "environment" },
{ observedOptionsKey: "traceName", promptLabel: "traceName" },
{ observedOptionsKey: "name", promptLabel: "name" },
+ {
+ observedOptionsKey: "providedModelName",
+ promptLabel: "providedModelName (model)",
+ },
+ { observedOptionsKey: "promptName", promptLabel: "promptName" },
+ { observedOptionsKey: "release", promptLabel: "release" },
+ {
+ observedOptionsKey: "experimentName",
+ promptLabel: "experimentName",
+ },
{ observedOptionsKey: "traceTags", promptLabel: "tags" },
{
observedOptionsKey: "calledToolNames",
@@ -30,6 +40,14 @@ export const RULE_FIELD_REGISTRY = fieldRegistryFromColumns(
],
fields: {
environment: { aliases: ["env"] },
+ statusMessage: {
+ aliases: ["statusmessage", "status_message", "status"],
+ },
+ providedModelName: {
+ aliases: ["providedmodelname", "provided_model_name", "model"],
+ },
+ promptName: { aliases: ["promptname", "prompt_name", "prompt"] },
+ promptVersion: { aliases: ["promptversion", "prompt_version"] },
traceName: { aliases: ["tracename", "trace_name"] },
userId: { aliases: ["userid", "user_id", "user"] },
sessionId: { aliases: ["sessionid", "session_id", "session"] },
@@ -48,6 +66,9 @@ export const RULE_FIELD_REGISTRY = fieldRegistryFromColumns(
negatedLabel: "Is not root observation",
},
experimentId: { aliases: ["experimentid", "experiment_id"] },
+ experimentName: {
+ aliases: ["experimentname", "experiment_name", "experiment"],
+ },
isExperimentItemRootSpan: {
aliases: [
"isexperimentitemrootspan",
diff --git a/web/src/features/evals/v2/fns/rules/classifySampleFiltersForRule.clienttest.ts b/web/src/features/evals/v2/fns/rules/classifySampleFiltersForRule.clienttest.ts
index b2338379a9..9f8c73bcdb 100644
--- a/web/src/features/evals/v2/fns/rules/classifySampleFiltersForRule.clienttest.ts
+++ b/web/src/features/evals/v2/fns/rules/classifySampleFiltersForRule.clienttest.ts
@@ -35,6 +35,52 @@ describe("classifySampleFiltersForRule", () => {
expect(result.unsupportedReasons.get(2)).toMatch(/scores/i);
});
+ it("supports filters backed directly by the evaluation payload", () => {
+ const filters = [
+ {
+ column: "providedModelName",
+ type: "string" as const,
+ operator: "=" as const,
+ value: "gpt-4o",
+ },
+ {
+ column: "promptName",
+ type: "string" as const,
+ operator: "=" as const,
+ value: "support-agent",
+ },
+ {
+ column: "promptVersion",
+ type: "number" as const,
+ operator: ">=" as const,
+ value: 2,
+ },
+ {
+ column: "release",
+ type: "string" as const,
+ operator: "=" as const,
+ value: "2026-08",
+ },
+ {
+ column: "statusMessage",
+ type: "string" as const,
+ operator: "contains" as const,
+ value: "rate limit",
+ },
+ {
+ column: "experimentName",
+ type: "string" as const,
+ operator: "=" as const,
+ value: "checkout-eval",
+ },
+ ] satisfies FilterState;
+
+ const result = classifySampleFiltersForRule(filters);
+
+ expect(result.supportedFilters).toEqual(filters);
+ expect(result.unsupportedReasons.size).toBe(0);
+ });
+
it("allows every filter to be excluded from the rule", () => {
const filters = [
{
diff --git a/web/src/features/search-bar/lib/searchBarInvariants.clienttest.ts b/web/src/features/search-bar/lib/searchBarInvariants.clienttest.ts
index c592ae6739..57c87d47e8 100644
--- a/web/src/features/search-bar/lib/searchBarInvariants.clienttest.ts
+++ b/web/src/features/search-bar/lib/searchBarInvariants.clienttest.ts
@@ -196,6 +196,31 @@ describe("search bar invariants — evaluation rules registry", () => {
filters: [{ column: "tags", type: "arrayOptions" }],
});
+ expect(
+ planCommit("model:gpt-4o", undefined, RULE_FIELD_REGISTRY),
+ ).toMatchObject({
+ status: "committed",
+ filters: [{ column: "providedModelName", type: "string" }],
+ });
+ expect(
+ planCommit("prompt:support-agent", undefined, RULE_FIELD_REGISTRY),
+ ).toMatchObject({
+ status: "committed",
+ filters: [{ column: "promptName", type: "string" }],
+ });
+ expect(
+ planCommit("status:rate-limit", undefined, RULE_FIELD_REGISTRY),
+ ).toMatchObject({
+ status: "committed",
+ filters: [{ column: "statusMessage", type: "string" }],
+ });
+ expect(
+ planCommit("experiment:checkout", undefined, RULE_FIELD_REGISTRY),
+ ).toMatchObject({
+ status: "committed",
+ filters: [{ column: "experimentName", type: "string" }],
+ });
+
expect(
planCommit("dataset:dataset-id", undefined, RULE_FIELD_REGISTRY),
).toMatchObject({