πͺ’ Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(experiments): show score filters in the search bar (#17209)
Nikita Kabardine2c5a0df3a92
diff --git a/web/src/features/experiments/components/table/ExperimentsTable.tsx b/web/src/features/experiments/components/table/ExperimentsTable.tsxindex 52884bf54a..21ef7d21ba 100644--- a/web/src/features/experiments/components/table/ExperimentsTable.tsx+++ b/web/src/features/experiments/components/table/ExperimentsTable.tsx@@ -13,7 +13,8 @@ import {useSidebarFilterState,} from "@/src/features/filters";import { usePaginationState } from "@/src/hooks/usePaginationState";-import { EXPERIMENTS_FIELD_REGISTRY } from "@/src/features/experiments/constants/experimentsSearchRegistry";+import { experimentsFieldRegistry } from "@/src/features/experiments/constants/experimentsSearchRegistry";+import { awaitsDatasetNames } from "@/src/features/experiments/fns/awaitsDatasetNames";import { withDatasetNamesResolved } from "@/src/features/experiments/fns/datasetNameFilter";import {DEFAULT_SEARCH_TYPE,@@ -346,6 +347,7 @@ export default function ExperimentsTable({filterOptions,datasetIdByName,datasetNameById,+ datasetNamesQuery,isFilterOptionsPending,} = useExperimentFilterOptions({projectId,@@ -396,8 +398,14 @@ export default function ExperimentsTable({);// Grammar search bar: an ADDITIONAL editor over the same FilterState the- // facet sidebar edits. Score filtering stays in the sidebar here β see- // experimentsSearchRegistry.+ // facet sidebar edits. Scoped to the facets this instance renders β a+ // dataset-scoped page omits the Dataset facet, and a token for a stripped+ // column would vanish silently.+ const searchRegistry = useMemo(+ () => experimentsFieldRegistry(filterConfig),+ [filterConfig],+ );+const observedOptions = useMemo(() => toObservedOptions(filterOptions, isFilterOptionsPending),[filterOptions, isFilterOptionsPending],@@ -420,7 +428,7 @@ export default function ExperimentsTable({setFilterState: setFiltersWrapper,setSearchQuery: noSearchLane,setSearchType: noSearchLane,- registry: EXPERIMENTS_FIELD_REGISTRY,+ registry: searchRegistry,});const combinedFilterState = queryFilter.filterState.concat(@@ -436,6 +444,12 @@ export default function ExperimentsTable({);// Use the custom hook for experiments data fetching+ // A dataset-name filter cannot be queried until the name -> id map lands.+ const waitingForDatasetNames = awaitsDatasetNames(+ combinedFilterState,+ datasetNamesQuery,+ );+const {experiments,totalCount,@@ -448,6 +462,7 @@ export default function ExperimentsTable({filterState,orderByState,paginationState,+ enabled: !waitingForDatasetNames,});// A score column that is empty for every experiment in view is noise, so only@@ -915,7 +930,7 @@ export default function ExperimentsTable({commit={searchBarCommit}observed={observedOptions}onApplyFilters={searchBarApplyFilters}- registry={EXPERIMENTS_FIELD_REGISTRY}+ registry={searchRegistry}/>{/* Toolbar spanning full width */}<DataTableToolbardiff --git a/web/src/features/experiments/constants/experimentsSearchRegistry.clienttest.ts b/web/src/features/experiments/constants/experimentsSearchRegistry.clienttest.tsnew file mode 100644index 0000000000..789ef26d6a--- /dev/null+++ b/web/src/features/experiments/constants/experimentsSearchRegistry.clienttest.ts@@ -0,0 +1,37 @@+// @vitest-environment node++import { describe, expect, it } from "vitest";++import { getExperimentsFilterConfig } from "@/src/features/experiments/components/table/filter-config";+import {+ EXPERIMENTS_FIELD_REGISTRY,+ experimentsFieldRegistry,+} from "./experimentsSearchRegistry";++describe("experimentsFieldRegistry β scoped to the facets it renders", () => {+ it("drops the dataset field when the page pins the dataset", () => {+ // A dataset-scoped page omits the Dataset facet, and the sidebar strips any+ // filter on a facet-less column on write β so offering `dataset:` there+ // would take the token and silently discard it.+ const scoped = experimentsFieldRegistry(+ getExperimentsFilterConfig(["experimentDatasetId"]),+ );++ expect(scoped.resolveField("dataset")).toBeNull();+ expect(scoped.resolveField("experimentDatasetName")).toBeNull();+ expect(scoped.resolveField("name")).not.toBeNull();+ });++ it("stops advertising an example the scoped registry cannot resolve", () => {+ const scoped = experimentsFieldRegistry(+ getExperimentsFilterConfig(["experimentDatasetId"]),+ );++ expect(scoped.searchExamples).not.toContain("dataset:legal-answer-quality");+ expect(scoped.searchExamples).toContain("name:sonnet");+ // The unscoped registry is unchanged: it has the facet, so it keeps it.+ expect(EXPERIMENTS_FIELD_REGISTRY.searchExamples).toContain(+ "dataset:legal-answer-quality",+ );+ });+});diff --git a/web/src/features/experiments/constants/experimentsSearchRegistry.ts b/web/src/features/experiments/constants/experimentsSearchRegistry.tsindex 108f96c879..a6e38b8988 100644--- a/web/src/features/experiments/constants/experimentsSearchRegistry.ts+++ b/web/src/features/experiments/constants/experimentsSearchRegistry.ts@@ -68,25 +68,57 @@ const EXPERIMENT_FIELD_OVERLAY = {},};-export const EXPERIMENTS_FIELD_REGISTRY: FieldRegistry =- fieldRegistryFromColumns(facetColumns(experimentsFilterConfig), {+/** Authored per view, never derived from the field ids β see the search-bar README. */+const SEARCH_EXAMPLES = [+ "dataset:legal-answer-quality",+ "name:sonnet",+ "scores.groundedness:>0.8",+];++/**+ * Built from the config the table actually renders, not the static one: a+ * dataset-scoped page omits the Dataset facet, and the sidebar then strips any+ * filter on that column on write. A registry that still offered `dataset:`+ * there would accept the token and let it be dropped without a word.+ */+export const experimentsFieldRegistry = (+ config: FilterConfig = experimentsFilterConfig,+): FieldRegistry => {+ const overlay = {id: "experiments",metadata: true,- // See SCORE_COLUMN_IDS: `scores.` needs the canonical column names, which- // the unification change introduces.- scores: false,+ // `scores.<name>` lowers onto the canonical `scores_avg` /+ // `score_categories` / `score_booleans` columns, which is exactly what this+ // view filters on since the score unification β so the bar can render a+ // score filter the sidebar set, and parse one the user types. `traceScores.`+ // stays closed: the level-agnostic columns already match at trace level, and+ // the `trace_*` columns are no longer offered.+ scores: true,traceScores: false,allowFreeText: false,// `name` is the only text column, and it is what people look a run up by.defaultTextField: "name",recentSearches: true,- // `dataset:` takes the dataset ID, not its name β the observed-value picker- // offers the ids, so the examples must not imply a name works.- searchExamples: [- "dataset:legal-answer-quality",- "name:sonnet",- 'metadata."model":opus',- ],aiContextFields: AI_CONTEXT_FIELDS,fields: EXPERIMENT_FIELD_OVERLAY,+ } as const;++ // The examples are authored, never generated β but a scoped config drops+ // facets, and an example naming a dropped one would advertise a field the bar+ // rejects. Built once to resolve them (aliases and `scores.` included), then+ // rebuilt with the examples that survive.+ const probe = fieldRegistryFromColumns(facetColumns(config), overlay);++ return fieldRegistryFromColumns(facetColumns(config), {+ ...overlay,+ searchExamples: SEARCH_EXAMPLES.filter((example) => {+ const separator = example.indexOf(":");+ if (separator === -1) return true;+ return probe.resolveField(example.slice(0, separator)) !== null;+ }),});+};++/** The unscoped registry: the server's AI-filter guard and the grammar tests. */+export const EXPERIMENTS_FIELD_REGISTRY: FieldRegistry =+ experimentsFieldRegistry();diff --git a/web/src/features/experiments/fns/awaitsDatasetNames.clienttest.ts b/web/src/features/experiments/fns/awaitsDatasetNames.clienttest.tsnew file mode 100644index 0000000000..6f2a37954c--- /dev/null+++ b/web/src/features/experiments/fns/awaitsDatasetNames.clienttest.ts@@ -0,0 +1,38 @@+import { describe, expect, it } from "vitest";+import type { FilterState } from "@langfuse/shared";++import { awaitsDatasetNames } from "./awaitsDatasetNames";++const nameFilter: FilterState = [+ {+ column: "experimentDatasetName",+ type: "stringOptions",+ operator: "any of",+ value: ["legal-answer-quality"],+ },+];++describe("awaitsDatasetNames", () => {+ it("waits while the names query is unsettled", () => {+ expect(+ awaitsDatasetNames(nameFilter, { isSuccess: false, isError: false }),+ ).toBe(true);+ });++ it("lets the query through once the names settle, empty or failed", () => {+ // Both states leave the name -> id map empty, which is why the gate cannot+ // be keyed on the map: either one would hold the table loading forever.+ expect(+ awaitsDatasetNames(nameFilter, { isSuccess: true, isError: false }),+ ).toBe(false);+ expect(+ awaitsDatasetNames(nameFilter, { isSuccess: false, isError: true }),+ ).toBe(false);+ });++ it("never waits without a dataset-name filter", () => {+ expect(awaitsDatasetNames([], { isSuccess: false, isError: false })).toBe(+ false,+ );+ });+});diff --git a/web/src/features/experiments/fns/awaitsDatasetNames.ts b/web/src/features/experiments/fns/awaitsDatasetNames.tsnew file mode 100644index 0000000000..b0636f495b--- /dev/null+++ b/web/src/features/experiments/fns/awaitsDatasetNames.ts@@ -0,0 +1,19 @@+import type { FilterState } from "@langfuse/shared";++const DATASET_NAME_COLUMN = "experimentDatasetName";++/**+ * Whether a dataset-NAME filter still cannot be translated to an id, so the+ * query has to wait. Keyed on the names query having SETTLED, never on the+ * name -> id map being non-empty: a project with no datasets and a failed fetch+ * both settle with an empty map, and gating on the DATA would hold the table in+ * its loading skeleton forever. Once settled, an untranslatable name+ * legitimately matches no rows.+ */+export function awaitsDatasetNames(+ filters: FilterState,+ datasetNames: { isSuccess: boolean; isError: boolean },+): boolean {+ if (datasetNames.isSuccess || datasetNames.isError) return false;+ return filters.some((filter) => filter.column === DATASET_NAME_COLUMN);+}diff --git a/web/src/features/experiments/hooks/useExperimentFilterOptions.ts b/web/src/features/experiments/hooks/useExperimentFilterOptions.tsindex 8a0626be99..29e41f7c95 100644--- a/web/src/features/experiments/hooks/useExperimentFilterOptions.ts+++ b/web/src/features/experiments/hooks/useExperimentFilterOptions.ts@@ -106,10 +106,19 @@ export function useExperimentFilterOptions({};}, [usedDatasets, filterOptions.data]);+ // The SETTLE state of the names query, not its contents: callers that must+ // wait for a name -> id translation cannot read that off the map, which is+ // empty both while loading and for a project with no datasets.+ const datasetNamesQuery = useMemo(+ () => ({ isSuccess: datasets.isSuccess, isError: datasets.isError }),+ [datasets.isSuccess, datasets.isError],+ );+return {filterOptions: experimentFilterOptions,datasetIdByName,datasetNameById,+ datasetNamesQuery,isFilterOptionsPending: datasets.isLoading || filterOptions.isLoading,};}diff --git a/web/src/features/experiments/hooks/useExperimentsTableData.ts b/web/src/features/experiments/hooks/useExperimentsTableData.tsindex f026a0c647..9819eac2be 100644--- a/web/src/features/experiments/hooks/useExperimentsTableData.ts+++ b/web/src/features/experiments/hooks/useExperimentsTableData.ts@@ -37,6 +37,13 @@ type UseExperimentsTableDataParams = {page: number;limit: number;};+ /**+ * Hold the queries while a filter value still needs translating. A dataset+ * NAME reaches the query as an id, so firing before the name -> id map lands+ * would query a bogus id and show "No experiments" before flipping to the+ * real rows.+ */+ enabled?: boolean;orderByState: {column: string;order: "ASC" | "DESC";@@ -48,6 +55,7 @@ export function useExperimentsTableData({filterState,paginationState,orderByState,+ enabled = true,}: UseExperimentsTableDataParams) {// Prepare query payloadsconst getCountPayload = useMemo(@@ -78,15 +86,17 @@ export function useExperimentsTableData({// on a cold load and the rejected zod input surfaces as a "Bad Request" toast.const isProjectReady = Boolean(projectId);+ const canQuery = isProjectReady && enabled;+// Fetch experimentsconst experimentsQuery = api.experiments.all.useQuery(getAllPayload, {- enabled: isProjectReady,+ enabled: canQuery,refetchOnWindowFocus: true,});// Fetch total countconst totalCountQuery = api.experiments.countAll.useQuery(getCountPayload, {- enabled: isProjectReady,+ enabled: canQuery,refetchOnWindowFocus: true,});@@ -99,7 +109,7 @@ export function useExperimentsTableData({const mostRecentQuery = api.experiments.mostRecent.useQuery({ ...getCountPayload, limit: MOST_RECENT_FALLBACK_LIMIT },{- enabled: isProjectReady && isEmptyWindow,+ enabled: canQuery && isEmptyWindow,refetchOnWindowFocus: false,},);@@ -132,7 +142,7 @@ export function useExperimentsTableData({// Fetch metricsconst metricsQuery = api.experiments.metrics.useQuery(metricsPayload!, {- enabled: isProjectReady && metricsPayload !== null,+ enabled: canQuery && metricsPayload !== null,refetchOnWindowFocus: false,staleTime: 0,});@@ -140,9 +150,9 @@ export function useExperimentsTableData({// Memoize joined data to prevent infinite re-renders// Handle loading, error, and success statesconst joinedData = useMemo(() => {- // A disabled query is neither loading nor errored, so keep the table in its- // loading state until the project id arrives instead of flashing "no rows".- if (!isProjectReady || experimentsQuery.isLoading) {+ // A disabled query is neither loading nor errored, so stay in the loading+ // state while the gate is shut instead of flashing "no rows".+ if (!canQuery || experimentsQuery.isLoading) {return { status: "loading" as const, rows: undefined };}@@ -161,7 +171,7 @@ export function useExperimentsTableData({metricsQuery.data,);}, [- isProjectReady,+ canQuery,experimentsQuery.isLoading,experimentsQuery.isError,isEmptyWindow,diff --git a/web/src/features/search-bar/lib/searchBarInvariants.clienttest.ts b/web/src/features/search-bar/lib/searchBarInvariants.clienttest.tsindex d4993020f5..4290ed1bda 100644--- a/web/src/features/search-bar/lib/searchBarInvariants.clienttest.ts+++ b/web/src/features/search-bar/lib/searchBarInvariants.clienttest.ts@@ -763,9 +763,9 @@ const experimentsView: RegistryUnderTest = {scoreContexts: [],fieldValues: ["x", "sonnet", "5", "0.8", "a b"],freeTextValues: ["hello", "run 12", "or", "!important"],- // Scores stay in the sidebar on this view, and after the score unification- // they sit on the SAME canonical columns the bar's `scores.` path recognizes β- // so the serializer has to be told they are not the bar's to render.+ // Score filters are the bar's to render on this view, so these must derive to+ // valid text rather than being skipped. The `trace_*` columns are not, and+ // stay sidebar-only.sidebarFilters: [[{@@ -846,14 +846,19 @@ describe("search bar invariants β experiments registry", () => {).toBeNull();});- it("keeps score dot-paths closed until the columns are unified", () => {- // The adapter lowers `scores.<name>` onto the canonical scores_avg /- // score_categories / score_booleans columns. Experiments names its score- // columns obs_* / trace_*, so an open `scores.` here would emit a filter on- // a column this view does not have.+ it("opens `scores.` and keeps `traceScores.` closed", () => {+ // `scores.<name>` lowers onto the canonical scores_avg / score_categories /+ // score_booleans columns, which is what this view filters on. Those columns+ // already match at trace level, so a separate `traceScores.` namespace would+ // offer a second way to say the same thing β against `trace_*` columns the+ // sidebar no longer offers.expect(EXPERIMENTS_FIELD_REGISTRY.resolveField("scores.groundedness"),- ).toBeNull();+ ).toEqual({+ type: "scores",+ key: "groundedness",+ level: "observation",+ });expect(EXPERIMENTS_FIELD_REGISTRY.resolveField("traceScores.x")).toBeNull();// Metadata is unaffected β that column exists under its canonical name.expect(EXPERIMENTS_FIELD_REGISTRY.resolveField("metadata.owner")).toEqual({@@ -862,6 +867,28 @@ describe("search bar invariants β experiments registry", () => {});});+ it("renders a score filter the sidebar set", () => {+ // The sidebar and the bar edit ONE filter state, so a score filter added in+ // the sidebar has to show up as a token β otherwise the bar reads as though+ // no score filter were applied.+ const derived = filterStateToQueryText(+ [+ {+ type: "numberObject",+ column: "scores_avg",+ key: "citation_usable_rate",+ operator: "=",+ value: 4,+ },+ ],+ {},+ EXPERIMENTS_FIELD_REGISTRY,+ );++ expect(derived.text).toBe("scores.citation_usable_rate:4");+ expect(derived.skippedFilters).toEqual([]);+ });+it("rewrites a bare word onto the experiment name", () => {const committed = planCommit("refund eval",