🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(evals): add payload-backed evaluation rule filters (#16484)
Tobias Wochingerc2931241cb4d
diff --git a/packages/shared/src/features/evals/observationForEval.test.ts b/packages/shared/src/features/evals/observationForEval.test.tsindex bf9dd78aa8..62d1142b1a 100644--- a/packages/shared/src/features/evals/observationForEval.test.ts+++ b/packages/shared/src/features/evals/observationForEval.test.ts@@ -83,6 +83,36 @@ describe("zipObservationToolCalls", () => {});describe("mapEventEvalFilterColumnIdToField", () => {+ it("maps filters backed directly by the evaluation payload", () => {+ const observation = {+ provided_model_name: "gpt-4o",+ prompt_name: "support-agent",+ prompt_version: 2,+ release: "2026-08",+ status_message: "rate limit exceeded",+ experiment_name: "checkout-eval",+ } as ObservationForEval;++ expect(+ mapEventEvalFilterColumnIdToField(observation, "providedModelName"),+ ).toBe("gpt-4o");+ expect(mapEventEvalFilterColumnIdToField(observation, "promptName")).toBe(+ "support-agent",+ );+ expect(+ mapEventEvalFilterColumnIdToField(observation, "promptVersion"),+ ).toBe(2);+ expect(mapEventEvalFilterColumnIdToField(observation, "release")).toBe(+ "2026-08",+ );+ expect(+ mapEventEvalFilterColumnIdToField(observation, "statusMessage"),+ ).toBe("rate limit exceeded");+ expect(+ mapEventEvalFilterColumnIdToField(observation, "experimentName"),+ ).toBe("checkout-eval");+ });+it("maps the experiment root filter to a boolean", () => {expect(mapEventEvalFilterColumnIdToField(diff --git a/packages/shared/src/features/evals/observationForEval.ts b/packages/shared/src/features/evals/observationForEval.tsindex 9be52be8b1..726753e6e7 100644--- a/packages/shared/src/features/evals/observationForEval.ts+++ b/packages/shared/src/features/evals/observationForEval.ts@@ -175,12 +175,18 @@ export type ObservationEvalFilterColumnInternal =| "name"| "environment"| "level"+ | "status_message"| "version"+ | "release"| "trace_name"| "user_id"| "session_id"+ | "provided_model_name"+ | "prompt_name"+ | "prompt_version"| "tags"| "experiment_id"+ | "experiment_name"| "experiment_dataset_id"| "experiment_item_root_span_id"| "metadata"@@ -347,6 +353,41 @@ export const observationEvalFilterColumns: ObservationEvalColumnDef[] = [internal: "version",nullable: true,},+ {+ name: "Release",+ id: "release",+ type: "string",+ internal: "release",+ nullable: true,+ },+ {+ name: "Status Message",+ id: "statusMessage",+ type: "string",+ internal: "status_message",+ nullable: true,+ },+ {+ name: "Provided Model Name",+ id: "providedModelName",+ type: "string",+ internal: "provided_model_name",+ nullable: true,+ },+ {+ name: "Prompt Name",+ id: "promptName",+ type: "string",+ internal: "prompt_name",+ nullable: true,+ },+ {+ name: "Prompt Version",+ id: "promptVersion",+ type: "number",+ internal: "prompt_version",+ nullable: true,+ },{name: "Trace Name",id: "traceName",@@ -403,6 +444,13 @@ export const observationEvalFilterColumns: ObservationEvalColumnDef[] = [options: [], // to be filled at runtimenullable: true,},+ {+ name: "Experiment Name",+ id: "experimentName",+ type: "string",+ internal: "experiment_name",+ nullable: true,+ },{name: "Is Experiment Item Root Span",id: "isExperimentItemRootSpan",diff --git a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.tsindex 5d04e00b55..b9fedc4517 100644--- a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts+++ b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts@@ -474,6 +474,7 @@ const FIELD_SETS = {"toolCalls","toolCallNames","experimentId",+ "experimentName","experimentItemRootSpanId","experimentItemExpectedOutput","experimentItemMetadata",diff --git a/packages/shared/src/server/queries/clickhouse-sql/events-stream-query.test.ts b/packages/shared/src/server/queries/clickhouse-sql/events-stream-query.test.tsindex 279b4aa92e..c28417a7c8 100644--- a/packages/shared/src/server/queries/clickhouse-sql/events-stream-query.test.ts+++ b/packages/shared/src/server/queries/clickhouse-sql/events-stream-query.test.ts@@ -68,6 +68,7 @@ describe("buildEventsStreamQuery", () => {expect(query).toContain("e.project_id = {projectId: String}");expect(query).toContain("e.is_deleted = 0");expect(query).toContain('e.is_app_root as "is_app_root"');+ expect(query).toContain('e.experiment_name as "experiment_name"');const orderIndex = query.lastIndexOf("ORDER BY ");const deduplicationIndex = query.lastIndexOf("LIMIT 1 BY ");diff --git a/packages/shared/src/server/repositories/events-stream.ts b/packages/shared/src/server/repositories/events-stream.tsindex 0e59db68de..c97fa15ab3 100644--- a/packages/shared/src/server/repositories/events-stream.ts+++ b/packages/shared/src/server/repositories/events-stream.ts@@ -73,6 +73,7 @@ export const getEventsStreamForEval = async (props: {output: unknown;metadata: Record<string, unknown> | null;experiment_id: string | null;+ experiment_name: string | null;experiment_item_root_span_id: string | null;experiment_item_expected_output: string | null;experiment_item_metadata: Record<string, unknown> | null;diff --git a/web/src/features/evals/v2/constants/ruleSearchRegistry.ts b/web/src/features/evals/v2/constants/ruleSearchRegistry.tsindex 09d43cb849..012c9e5721 100644--- a/web/src/features/evals/v2/constants/ruleSearchRegistry.ts+++ b/web/src/features/evals/v2/constants/ruleSearchRegistry.ts@@ -18,6 +18,16 @@ export const RULE_FIELD_REGISTRY = fieldRegistryFromColumns({ observedOptionsKey: "environment", promptLabel: "environment" },{ observedOptionsKey: "traceName", promptLabel: "traceName" },{ observedOptionsKey: "name", promptLabel: "name" },+ {+ observedOptionsKey: "providedModelName",+ promptLabel: "providedModelName (model)",+ },+ { observedOptionsKey: "promptName", promptLabel: "promptName" },+ { observedOptionsKey: "release", promptLabel: "release" },+ {+ observedOptionsKey: "experimentName",+ promptLabel: "experimentName",+ },{ observedOptionsKey: "traceTags", promptLabel: "tags" },{observedOptionsKey: "calledToolNames",@@ -30,6 +40,14 @@ export const RULE_FIELD_REGISTRY = fieldRegistryFromColumns(],fields: {environment: { aliases: ["env"] },+ statusMessage: {+ aliases: ["statusmessage", "status_message", "status"],+ },+ providedModelName: {+ aliases: ["providedmodelname", "provided_model_name", "model"],+ },+ promptName: { aliases: ["promptname", "prompt_name", "prompt"] },+ promptVersion: { aliases: ["promptversion", "prompt_version"] },traceName: { aliases: ["tracename", "trace_name"] },userId: { aliases: ["userid", "user_id", "user"] },sessionId: { aliases: ["sessionid", "session_id", "session"] },@@ -48,6 +66,9 @@ export const RULE_FIELD_REGISTRY = fieldRegistryFromColumns(negatedLabel: "Is not root observation",},experimentId: { aliases: ["experimentid", "experiment_id"] },+ experimentName: {+ aliases: ["experimentname", "experiment_name", "experiment"],+ },isExperimentItemRootSpan: {aliases: ["isexperimentitemrootspan",diff --git a/web/src/features/evals/v2/fns/rules/classifySampleFiltersForRule.clienttest.ts b/web/src/features/evals/v2/fns/rules/classifySampleFiltersForRule.clienttest.tsindex b2338379a9..9f8c73bcdb 100644--- a/web/src/features/evals/v2/fns/rules/classifySampleFiltersForRule.clienttest.ts+++ b/web/src/features/evals/v2/fns/rules/classifySampleFiltersForRule.clienttest.ts@@ -35,6 +35,52 @@ describe("classifySampleFiltersForRule", () => {expect(result.unsupportedReasons.get(2)).toMatch(/scores/i);});+ it("supports filters backed directly by the evaluation payload", () => {+ const filters = [+ {+ column: "providedModelName",+ type: "string" as const,+ operator: "=" as const,+ value: "gpt-4o",+ },+ {+ column: "promptName",+ type: "string" as const,+ operator: "=" as const,+ value: "support-agent",+ },+ {+ column: "promptVersion",+ type: "number" as const,+ operator: ">=" as const,+ value: 2,+ },+ {+ column: "release",+ type: "string" as const,+ operator: "=" as const,+ value: "2026-08",+ },+ {+ column: "statusMessage",+ type: "string" as const,+ operator: "contains" as const,+ value: "rate limit",+ },+ {+ column: "experimentName",+ type: "string" as const,+ operator: "=" as const,+ value: "checkout-eval",+ },+ ] satisfies FilterState;++ const result = classifySampleFiltersForRule(filters);++ expect(result.supportedFilters).toEqual(filters);+ expect(result.unsupportedReasons.size).toBe(0);+ });+it("allows every filter to be excluded from the rule", () => {const filters = [{diff --git a/web/src/features/search-bar/lib/searchBarInvariants.clienttest.ts b/web/src/features/search-bar/lib/searchBarInvariants.clienttest.tsindex c592ae6739..57c87d47e8 100644--- a/web/src/features/search-bar/lib/searchBarInvariants.clienttest.ts+++ b/web/src/features/search-bar/lib/searchBarInvariants.clienttest.ts@@ -196,6 +196,31 @@ describe("search bar invariants — evaluation rules registry", () => {filters: [{ column: "tags", type: "arrayOptions" }],});+ expect(+ planCommit("model:gpt-4o", undefined, RULE_FIELD_REGISTRY),+ ).toMatchObject({+ status: "committed",+ filters: [{ column: "providedModelName", type: "string" }],+ });+ expect(+ planCommit("prompt:support-agent", undefined, RULE_FIELD_REGISTRY),+ ).toMatchObject({+ status: "committed",+ filters: [{ column: "promptName", type: "string" }],+ });+ expect(+ planCommit("status:rate-limit", undefined, RULE_FIELD_REGISTRY),+ ).toMatchObject({+ status: "committed",+ filters: [{ column: "statusMessage", type: "string" }],+ });+ expect(+ planCommit("experiment:checkout", undefined, RULE_FIELD_REGISTRY),+ ).toMatchObject({+ status: "committed",+ filters: [{ column: "experimentName", type: "string" }],+ });+expect(planCommit("dataset:dataset-id", undefined, RULE_FIELD_REGISTRY),).toMatchObject({