πͺ’ Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(charts): apply metadata and prompt version filters to in-view charts (#18066)
Nikita Kabardin7d2f451c5bdb
diff --git a/web/src/__tests__/server/chart-view-filter-forwarding.servertest.ts b/web/src/__tests__/server/chart-view-filter-forwarding.servertest.tsnew file mode 100644index 0000000000..b276462e7f--- /dev/null+++ b/web/src/__tests__/server/chart-view-filter-forwarding.servertest.ts@@ -0,0 +1,135 @@+import { randomUUID } from "crypto";+import { type FilterState } from "@langfuse/shared";+import { executeQuery } from "@langfuse/shared/query/server";+import { createEvent, createEventsCh } from "@langfuse/shared/src/server";+import { env } from "@/src/env.mjs";+import { type ChartViewConfig } from "@/src/features/chart-view/types";+import { buildChartQuery } from "@/src/features/chart-view/lib/buildChartQuery";+import { toChartFilters } from "@/src/features/chart-view/lib/chartFilterCompatibility";++/**+ * The chart view narrows the table's filters with `toChartFilters` and hands+ * the rest to the v2 observations aggregate query. Which columns forward is a+ * hand-maintained policy list, so the two halves can drift apart silently: a+ * forwarded filter the query rejects breaks the whole chart, and a dropped+ * filter the query would have honoured makes the chart disagree with the table+ * it is supposed to visualise. These run the real query against ClickHouse to+ * pin that both directions hold for the shapes the events FilterState carries.+ *+ * v2 reads events_core/events_full, which only exist on a v4 deployment.+ */+const maybe =+ env.LANGFUSE_MIGRATION_V4_ALLOW_PREVIEW_OPT_IN === "true"+ ? describe+ : describe.skip;++const CONFIG: ChartViewConfig = {+ metric: "count",+ aggregation: "count",+ breakdown: "none",+ chartType: "NUMBER",+ timeGranularity: "hour",+};++maybe("chart view filter forwarding", () => {+ const projectId = randomUUID();+ const startTime = new Date();+ const fromTimestamp = new Date(startTime.getTime() - 60 * 60 * 1000);+ const toTimestamp = new Date(startTime.getTime() + 60 * 60 * 1000);++ /** The chart's own path: narrow the table filters, then run the aggregate. */+ const chartCount = async (filterState: FilterState): Promise<number> => {+ const rows = await executeQuery(+ projectId,+ buildChartQuery({+ config: CONFIG,+ filters: toChartFilters(filterState),+ fromTimestamp,+ toTimestamp,+ }),+ "v2",+ true,+ );+ return Number(rows[0]?.count_count ?? 0);+ };++ beforeAll(async () => {+ await createEventsCh([+ createEvent({+ project_id: projectId,+ trace_id: randomUUID(),+ start_time: startTime,+ metadata_names: ["tier"],+ metadata_values: ["gold"],+ prompt_version: 3,+ }),+ createEvent({+ project_id: projectId,+ trace_id: randomUUID(),+ start_time: startTime,+ metadata_names: ["tier"],+ metadata_values: ["silver"],+ prompt_version: 7,+ }),+ createEvent({+ project_id: projectId,+ trace_id: randomUUID(),+ start_time: startTime,+ metadata_names: [],+ metadata_values: [],+ prompt_version: null,+ }),+ ]);+ });++ it("charts every event when nothing is filtered", async () => {+ await expect(chartCount([])).resolves.toBe(3);+ });++ it("applies a keyed metadata filter", async () => {+ await expect(+ chartCount([+ {+ column: "metadata",+ type: "stringObject",+ key: "tier",+ operator: "=",+ value: "gold",+ },+ ]),+ ).resolves.toBe(1);+ });++ it("requires the key to be present on a negated metadata match", async () => {+ // The event without a `tier` key is not a "tier does not contain gold"+ // match β the same semantic the table applies.+ await expect(+ chartCount([+ {+ column: "metadata",+ type: "stringObject",+ key: "tier",+ operator: "does not contain",+ value: "gold",+ },+ ]),+ ).resolves.toBe(1);+ });++ it("applies the numeric promptVersion filter", async () => {+ await expect(+ chartCount([+ { column: "promptVersion", type: "number", operator: ">=", value: 5 },+ ]),+ ).resolves.toBe(1);+ });++ it("drops the filters it cannot forward instead of erroring the query", async () => {+ await expect(+ chartCount([+ { column: "latency", type: "number", operator: ">", value: 0 },+ { column: "userId", type: "null", operator: "is not null", value: "" },+ ]),+ ).resolves.toBe(3);+ });+});diff --git a/web/src/features/chart-view/lib/chartFilterCompatibility.clienttest.ts b/web/src/features/chart-view/lib/chartFilterCompatibility.clienttest.tsindex 6948abc4a4..20aa30c26c 100644--- a/web/src/features/chart-view/lib/chartFilterCompatibility.clienttest.ts+++ b/web/src/features/chart-view/lib/chartFilterCompatibility.clienttest.ts@@ -1,6 +1,8 @@// @vitest-environment nodeimport {+ chartConditionExclusionReason,+ chartFacetExclusionReason,chartFilterExclusionReason,chartSearchFieldReason,classifyChartFilters,@@ -25,12 +27,14 @@ describe("chartFilterExclusionReason", () => {"toolNames","experimentId","isRootObservation",+ "promptVersion",+ "metadata",]) {expect(chartFilterExclusionReason(col)).toBeNull();}});- it("groups measures, scores, comments, and metadata by reason", () => {+ it("groups measures, scores, and comments by reason", () => {expect(chartFilterExclusionReason("latency")).toMatch(/latency, cost/i);expect(chartFilterExclusionReason("cachedInputTokens")).toMatch(/latency, cost/i,@@ -43,11 +47,124 @@ describe("chartFilterExclusionReason", () => {expect(chartFilterExclusionReason("trace_score_categories")).toMatch(/scores/i,);- // promptVersion is a numeric field but the observations dimension is a- // string β forwarding it errors the query, so it is NOT forwardable.- expect(chartFilterExclusionReason("promptVersion")).not.toBeNull();expect(chartFilterExclusionReason("commentContent")).toMatch(/comments/i);- expect(chartFilterExclusionReason("metadata")).toMatch(/metadata/i);+ });+});++describe("chartConditionExclusionReason", () => {+ it("forwards a keyed metadata filter", () => {+ expect(+ chartConditionExclusionReason({+ column: "metadata",+ type: "stringObject",+ key: "langfuse_user_email",+ operator: "does not contain",+ value: "@langfuse.com",+ }),+ ).toBeNull();+ });++ it("excludes a metadata condition that is not the keyed stringObject shape", () => {+ // The query builder accepts metadata only as a keyed stringObject;+ // forwarding another shape errors the whole chart, not just one filter.+ expect(+ chartConditionExclusionReason({+ column: "metadata",+ type: "null",+ operator: "is not null",+ value: "",+ }),+ ).not.toBeNull();+ });++ it("excludes a presence check on an otherwise forwardable column", () => {+ expect(+ chartConditionExclusionReason({+ column: "userId",+ type: "null",+ operator: "is not null",+ value: "",+ }),+ ).toMatch(/is set/i);+ });++ it("keeps the column reason for an unsupported column", () => {+ expect(+ chartConditionExclusionReason({+ column: "latency",+ type: "number",+ operator: ">",+ value: 2,+ }),+ ).toMatch(/latency, cost/i);+ });+});++describe("chartFacetExclusionReason", () => {+ const metadataFilter = (type: "stringObject" | "string"): FilterState =>+ type === "stringObject"+ ? [+ {+ column: "metadata",+ type: "stringObject",+ key: "tier",+ operator: "=",+ value: "gold",+ },+ ]+ : // the shape a hand-edited URL can still decode into+ [{ column: "metadata", type: "string", operator: "=", value: "gold" }];++ it("leaves a facet live when its conditions forward", () => {+ expect(+ chartFacetExclusionReason(metadataFilter("stringObject"), "metadata"),+ ).toBeNull();+ });++ it("blocks a facet holding a condition the chart drops", () => {+ // Column-level policy says metadata is fine; this condition is not, and the+ // facet must say so rather than look applied.+ expect(chartFilterExclusionReason("metadata")).toBeNull();+ expect(+ chartFacetExclusionReason(metadataFilter("string"), "metadata"),+ ).not.toBeNull();+ });++ it("blocks a facet holding a presence check", () => {+ expect(+ chartFacetExclusionReason(+ [+ {+ column: "userId",+ type: "null",+ operator: "is not null",+ value: "",+ },+ ],+ "userId",+ ),+ ).toMatch(/is set/i);+ });++ it("falls back to the column policy for a facet with no condition", () => {+ expect(chartFacetExclusionReason([], "metadata")).toBeNull();+ expect(chartFacetExclusionReason([], "latency")).toMatch(/latency, cost/i);+ });++ it("ignores conditions on other columns", () => {+ expect(+ chartFacetExclusionReason(+ [+ {+ column: "userId",+ type: "null",+ operator: "is not null",+ value: "",+ },+ ],+ "metadata",+ ),+ ).toBeNull();});});@@ -73,6 +190,14 @@ describe("toChartFilters", () => {operator: "=",value: true,},+ {+ column: "metadata",+ type: "stringObject",+ key: "langfuse_user_email",+ operator: "does not contain",+ value: "@langfuse.com",+ },+ { column: "promptVersion", type: "number", operator: ">=", value: 2 },{column: "scores_avg",type: "numberObject",@@ -91,10 +216,19 @@ describe("toChartFilters", () => {"userId","tags","isRootObservation",+ "metadata",+ "promptVersion",]);// the rename keeps the rest of the filter intactconst tags = result.find((f) => f.column === "tags");expect(tags).toMatchObject({ operator: "all of", value: ["prod"] });+ // metadata forwards with the key the query builder needs+ expect(result.find((f) => f.column === "metadata")).toMatchObject({+ type: "stringObject",+ key: "langfuse_user_email",+ operator: "does not contain",+ value: "@langfuse.com",+ });});});@@ -105,6 +239,8 @@ describe("chartSearchFieldReason", () => {expect(chartSearchFieldReason("user")).toBeNull(); // alias -> userIdexpect(chartSearchFieldReason("tags")).toBeNull(); // alias -> traceTagsexpect(chartSearchFieldReason("model")).toBeNull(); // -> providedModelName+ expect(chartSearchFieldReason("promptVersion")).toBeNull();+ expect(chartSearchFieldReason("metadata.region")).toBeNull();});it("classifies unsupported grammar fields by group", () => {@@ -114,7 +250,6 @@ describe("chartSearchFieldReason", () => {expect(chartSearchFieldReason("traceScores.helpfulness")).toMatch(/scores/i,);- expect(chartSearchFieldReason("metadata.region")).toMatch(/metadata/i);// a search-bar startTime bound the chart can't honourexpect(chartSearchFieldReason("startTime")).toMatch(/this field/i);});diff --git a/web/src/features/chart-view/lib/chartFilterCompatibility.ts b/web/src/features/chart-view/lib/chartFilterCompatibility.tsindex 4f00691180..74ce08808a 100644--- a/web/src/features/chart-view/lib/chartFilterCompatibility.ts+++ b/web/src/features/chart-view/lib/chartFilterCompatibility.ts@@ -1,4 +1,4 @@-import { type FilterState } from "@langfuse/shared";+import { type FilterCondition, type FilterState } from "@langfuse/shared";import { resolveField } from "@/src/features/search-bar";/**@@ -6,11 +6,11 @@ import { resolveField } from "@/src/features/search-bar";* human reason for the ones it can't.** The chart runs the observations aggregate query (`dashboard.executeQuery`,- * v2 events read path β see `buildChartQuery`). That query accepts a subset of- * the columns the events table exposes as filters: the dimensions the widget- * builder also offers (LFE-10751). Everything else β measures, scores,- * comments, metadata, and a few structural columns β has no dimension to filter- * on, so it CANNOT be applied to the chart.+ * v2 events read path β see `buildChartQuery`). That query accepts the columns+ * the widget builder offers as dimensions, plus `metadata`, which the query+ * builder takes as a keyed filter rather than a dimension. Measures, scores,+ * comments and a few structural columns have nothing to filter on, so they+ * CANNOT be applied to the chart.** Rather than hide the chart when an unsupported filter is present (the old* all-or-nothing gate), we forward what we can and mark the rest as "not@@ -21,10 +21,12 @@ import { resolveField } from "@/src/features/search-bar";*//**- * Events-table filter columns whose values forward 1:1 onto an observations- * query dimension. Keyed by the `column` string the events `FilterState` uses.- * `traceTags` is the one whose query dimension name differs (`tags`) β see- * {@link CHART_FILTER_COLUMN_RENAME}.+ * Events-table filter columns the chart query honours. Keyed by the `column`+ * string the events `FilterState` uses. Most forward 1:1 onto an observations+ * query dimension; `traceTags` is renamed to its dimension name (`tags`, see+ * {@link CHART_FILTER_COLUMN_RENAME}) and `metadata` is not a dimension at all+ * (see {@link chartConditionExclusionReason}). Column membership alone is not+ * sufficient β a condition also has to carry a shape the query accepts.*/const FORWARDABLE_CHART_FILTER_COLUMNS: ReadonlySet<string> = new Set(["environment",@@ -38,11 +40,7 @@ const FORWARDABLE_CHART_FILTER_COLUMNS: ReadonlySet<string> = new Set(["version","release","promptName",- // NOTE: promptVersion is intentionally NOT forwardable. The search-bar- // grammar registers it as a numeric field, so `promptVersion:>2` lowers to a- // `number` filter β but the observations-view dimension is typed `string`,- // which rejects numeric filters and errors the chart query. It shows as "not- // applied" instead.+ "promptVersion","traceTags","toolNames","calledToolNames",@@ -50,6 +48,7 @@ const FORWARDABLE_CHART_FILTER_COLUMNS: ReadonlySet<string> = new Set(["experimentDatasetId","experimentId","isRootObservation",+ "metadata",]);/**@@ -92,6 +91,19 @@ const COMMENT_COLUMNS = new Set(["commentCount", "commentContent"]);export const CHART_SEARCH_QUERY_REASON ="Charts can't apply text search at the moment β still narrows the table.";+/**+ * Reason for a presence check (`has:`/`-has:`, a `null`-type filter). The+ * aggregate query reads the raw column, where an unset value is an empty string+ * or an empty array rather than NULL, so a presence check there does not select+ * the rows the table selects.+ */+const CHART_PRESENCE_FILTER_REASON =+ "Charts can't filter by whether a field is set at the moment β still applies to the table.";++/** Fallback for a column, or a filter shape, the chart query has no place for. */+const CHART_UNSUPPORTED_FIELD_REASON =+ "Charts can't filter by this field at the moment β still applies to the table.";+/*** The reason a filter on `column` is NOT applied to the chart, or `null` if it* is forwarded. User-facing hover copy: plain "not at the moment" framing β@@ -107,25 +119,57 @@ export function chartFilterExclusionReason(column: string): string | null {return "Charts can't filter by scores at the moment β still applies to the table.";if (COMMENT_COLUMNS.has(column))return "Charts can't filter by comments at the moment β still applies to the table.";- if (column === "metadata")- return "Charts can't filter by metadata at the moment β still applies to the table.";- return "Charts can't filter by this field at the moment β still applies to the table.";+ return CHART_UNSUPPORTED_FIELD_REASON;+}++/**+ * The reason a single condition is NOT applied to the chart, or `null` if it is+ * forwarded. Condition-level, because a forwardable column can still carry a+ * shape the aggregate query rejects or answers differently from the table:+ *+ * - a presence check on any column ({@link CHART_PRESENCE_FILTER_REASON});+ * - `metadata` in anything but its keyed `stringObject` form β that is the only+ * metadata shape the query builder accepts, and forwarding another one would+ * error the whole chart rather than one filter.+ */+export function chartConditionExclusionReason(+ filter: FilterCondition,+): string | null {+ const columnReason = chartFilterExclusionReason(filter.column);+ if (columnReason) return columnReason;+ if (filter.type === "null") return CHART_PRESENCE_FILTER_REASON;+ if (filter.column === "metadata" && filter.type !== "stringObject")+ return CHART_UNSUPPORTED_FIELD_REASON;+ return null;+}++/**+ * The reason a sidebar FACET is not applied to the chart, or `null` if it is.+ * A facet stands for a column but displays the conditions on it, so it reads+ * blocked as soon as one of them is not forwarded β otherwise a facet whose+ * column is forwardable would look applied while the chart quietly ignored the+ * condition it holds. With no condition yet, the column policy decides.+ */+export function chartFacetExclusionReason(+ filterState: FilterState,+ column: string,+): string | null {+ for (const filter of filterState) {+ if (filter.column !== column) continue;+ const reason = chartConditionExclusionReason(filter);+ if (reason) return reason;+ }+ return chartFilterExclusionReason(column);}/*** Narrows a `FilterState` to the subset the chart query can honour, renaming* the few columns whose observations-view dimension name differs. The inverse- * of {@link chartFilterExclusionReason} on the forwarding side.+ * of {@link chartConditionExclusionReason} on the forwarding side.*/export function toChartFilters(filterState: FilterState): FilterState {return filterState- .filter(- (f) =>- // Presence checks (`has:`/`-has:`, a `null` filter) aren't applied by- // the aggregate chart query β drop them so we never forward one and- // 422; the search bar marks them "not applied" (chartSearchFieldReason).- f.type !== "null" && FORWARDABLE_CHART_FILTER_COLUMNS.has(f.column),- )+ .filter((f) => chartConditionExclusionReason(f) === null).map((f) => {const renamed = CHART_FILTER_COLUMN_RENAME[f.column];return renamed ? { ...f, column: renamed } : f;@@ -141,17 +185,18 @@ export function toChartFilters(filterState: FilterState): FilterState {export function chartSearchFieldReason(fieldName: string): string | null {const ref = resolveField(fieldName);if (!ref) return null;+ // A metadata dot-path lowers to the keyed `stringObject` shape the chart+ // query accepts, so it deactivates only if the column policy says so.if (ref.type === "metadata") return chartFilterExclusionReason("metadata");if (ref.type === "scores")return chartFilterExclusionReason(ref.level === "trace" ? "trace_scores_avg" : "scores_avg",);- // `has:`/`-has:` presence checks lower to a null-check filter, which the chart- // doesn't apply (dropped by toChartFilters) β so the pill is deactivated too.if (ref.type === "searchScope" || (ref.type === "pseudo" && ref.id === "in"))return CHART_SEARCH_QUERY_REASON;- if (ref.type === "pseudo")- return "Charts can't filter by whether a field is set at the moment β still applies to the table.";+ // `has:`/`-has:` lowers to a presence check, which the chart doesn't apply+ // (dropped by toChartFilters) β so the pill is deactivated too.+ if (ref.type === "pseudo") return CHART_PRESENCE_FILTER_REASON;return chartFilterExclusionReason(ref.field.id);}@@ -166,7 +211,7 @@ export function classifyChartFilters(filterState: FilterState): {} {const excluded = new Map<string, string>();for (const f of filterState) {- const reason = chartFilterExclusionReason(f.column);+ const reason = chartConditionExclusionReason(f);if (reason) excluded.set(f.column, reason);}return { forwarded: toChartFilters(filterState), excluded };diff --git a/web/src/features/events/components/EventsTable.tsx b/web/src/features/events/components/EventsTable.tsxindex 2cbbd83615..5a660b8d32 100644--- a/web/src/features/events/components/EventsTable.tsx+++ b/web/src/features/events/components/EventsTable.tsx@@ -166,7 +166,7 @@ import { EventsChartView } from "@/src/features/chart-view/EventsChartView";import { ViewModeToggle, useChartViewState } from "@/src/features/chart-view";import { EventsOutlierStrip } from "@/src/features/events/components/outlier-strip/EventsOutlierStrip";import {- chartFilterExclusionReason,+ chartFacetExclusionReason,chartSearchFieldReason,CHART_SEARCH_QUERY_REASON,} from "@/src/features/chart-view/lib/chartFilterCompatibility";@@ -897,12 +897,20 @@ export default function ObservationsEventsTable({// The chart is actually on screen (not just enabled). Only then do we mark// the filters it can't honour as "not applied", so table mode stays untouched.- // Both surfaces use the stateless per-column / per-field reason resolvers- // (chartFilterExclusionReason / chartSearchFieldReason) β a filter deactivates- // in the sidebar and its search-bar pill identically.+ // Both surfaces read the same resolvers (chartFacetExclusionReason /+ // chartSearchFieldReason) β a filter deactivates in the sidebar and its+ // search-bar pill identically.const chartActive = chartEnabled && chartViewMode === "chart";// Free-text search is never applied to the chart (it has no aggregate form).const chartFreeTextIgnored = chartActive && Boolean(searchQuery);+ // A facet reads blocked when its column is unsupported OR when it holds a+ // condition the chart drops (a metadata filter outside its keyed shape, a+ // presence check), so the sidebar can never show a filter as applied that the+ // chart query left out.+ const chartBlockedColumnReason = useCallback(+ (column: string) => chartFacetExclusionReason(filterState, column),+ [filterState],+ );// Use the custom hook for observations data fetchingconst {@@ -1963,7 +1971,7 @@ export default function ObservationsEventsTable({queryFilter={queryFilter}filterWithAI={sidebarAiFiltersEnabled}blockedColumnReason={- chartActive ? chartFilterExclusionReason : undefined+ chartActive ? chartBlockedColumnReason : undefined}// inline: flow at natural height in the sheet's single scroll// (no internal ScrollArea). Desktop sidebar stays default.@@ -2216,10 +2224,9 @@ export default function ObservationsEventsTable({queryFilter={queryFilter}filterWithAI={sidebarAiFiltersEnabled}// In chart mode, block filters the chart can't apply β active or- // not β dimmed + hover reason. Stateless per-column resolver,- // matching the search bar.+ // not β dimmed + hover reason, matching the search bar.blockedColumnReason={- chartActive ? chartFilterExclusionReason : undefined+ chartActive ? chartBlockedColumnReason : undefined}/>)}diff --git a/web/src/features/events/components/outlier-strip/lib/filterCompatibility.clienttest.ts b/web/src/features/events/components/outlier-strip/lib/filterCompatibility.clienttest.tsindex 0e779f5535..a314c64ee2 100644--- a/web/src/features/events/components/outlier-strip/lib/filterCompatibility.clienttest.ts+++ b/web/src/features/events/components/outlier-strip/lib/filterCompatibility.clienttest.ts@@ -18,6 +18,20 @@ describe("canApplyOutlierStripFilters", () => {expect(canApplyOutlierStripFilters(filters, false)).toBe(true);});+ it("stays available under a keyed metadata filter", () => {+ const filters: FilterState = [+ {+ column: "metadata",+ type: "stringObject",+ key: "langfuse_user_email",+ operator: "does not contain",+ value: "@langfuse.com",+ },+ ];++ expect(canApplyOutlierStripFilters(filters, false)).toBe(true);+ });+it("rejects filters the aggregate query cannot represent", () => {const unsupportedFilters: FilterState[] = [[{ column: "latency", type: "number", operator: ">", value: 2 }],diff --git a/web/src/features/events/components/outlier-strip/lib/filterCompatibility.ts b/web/src/features/events/components/outlier-strip/lib/filterCompatibility.tsindex da95de5414..d6e659908b 100644--- a/web/src/features/events/components/outlier-strip/lib/filterCompatibility.ts+++ b/web/src/features/events/components/outlier-strip/lib/filterCompatibility.ts@@ -1,5 +1,5 @@import { type FilterState } from "@langfuse/shared";-import { chartFilterExclusionReason } from "@/src/features/chart-view/lib/chartFilterCompatibility";+import { chartConditionExclusionReason } from "@/src/features/chart-view/lib/chartFilterCompatibility";export const canApplyOutlierStripFilters = (filterState: FilterState,@@ -11,7 +11,6 @@ export const canApplyOutlierStripFilters = (if (filter.type === "datetime" && filter.column === "startTime") {return true;}- if (filter.type === "null") return false;- return chartFilterExclusionReason(filter.column) === null;+ return chartConditionExclusionReason(filter) === null;});};