Skip to content

πŸͺ’ Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(experiments): show score filters in the search bar (#17209)

Nikita Kabardine2c5a0df3a92
diff --git a/web/src/features/experiments/components/table/ExperimentsTable.tsx b/web/src/features/experiments/components/table/ExperimentsTable.tsx
index 52884bf54a..21ef7d21ba 100644
--- a/web/src/features/experiments/components/table/ExperimentsTable.tsx
+++ b/web/src/features/experiments/components/table/ExperimentsTable.tsx
@@ -13,7 +13,8 @@ import {
useSidebarFilterState,
} from "@/src/features/filters";
import { usePaginationState } from "@/src/hooks/usePaginationState";
-import { EXPERIMENTS_FIELD_REGISTRY } from "@/src/features/experiments/constants/experimentsSearchRegistry";
+import { experimentsFieldRegistry } from "@/src/features/experiments/constants/experimentsSearchRegistry";
+import { awaitsDatasetNames } from "@/src/features/experiments/fns/awaitsDatasetNames";
import { withDatasetNamesResolved } from "@/src/features/experiments/fns/datasetNameFilter";
import {
DEFAULT_SEARCH_TYPE,
@@ -346,6 +347,7 @@ export default function ExperimentsTable({
filterOptions,
datasetIdByName,
datasetNameById,
+ datasetNamesQuery,
isFilterOptionsPending,
} = useExperimentFilterOptions({
projectId,
@@ -396,8 +398,14 @@ export default function ExperimentsTable({
);
// Grammar search bar: an ADDITIONAL editor over the same FilterState the
- // facet sidebar edits. Score filtering stays in the sidebar here β€” see
- // experimentsSearchRegistry.
+ // facet sidebar edits. Scoped to the facets this instance renders β€” a
+ // dataset-scoped page omits the Dataset facet, and a token for a stripped
+ // column would vanish silently.
+ const searchRegistry = useMemo(
+ () => experimentsFieldRegistry(filterConfig),
+ [filterConfig],
+ );
+
const observedOptions = useMemo(
() => toObservedOptions(filterOptions, isFilterOptionsPending),
[filterOptions, isFilterOptionsPending],
@@ -420,7 +428,7 @@ export default function ExperimentsTable({
setFilterState: setFiltersWrapper,
setSearchQuery: noSearchLane,
setSearchType: noSearchLane,
- registry: EXPERIMENTS_FIELD_REGISTRY,
+ registry: searchRegistry,
});
const combinedFilterState = queryFilter.filterState.concat(
@@ -436,6 +444,12 @@ export default function ExperimentsTable({
);
// Use the custom hook for experiments data fetching
+ // A dataset-name filter cannot be queried until the name -> id map lands.
+ const waitingForDatasetNames = awaitsDatasetNames(
+ combinedFilterState,
+ datasetNamesQuery,
+ );
+
const {
experiments,
totalCount,
@@ -448,6 +462,7 @@ export default function ExperimentsTable({
filterState,
orderByState,
paginationState,
+ enabled: !waitingForDatasetNames,
});
// A score column that is empty for every experiment in view is noise, so only
@@ -915,7 +930,7 @@ export default function ExperimentsTable({
commit={searchBarCommit}
observed={observedOptions}
onApplyFilters={searchBarApplyFilters}
- registry={EXPERIMENTS_FIELD_REGISTRY}
+ registry={searchRegistry}
/>
{/* Toolbar spanning full width */}
<DataTableToolbar
diff --git a/web/src/features/experiments/constants/experimentsSearchRegistry.clienttest.ts b/web/src/features/experiments/constants/experimentsSearchRegistry.clienttest.ts
new file mode 100644
index 0000000000..789ef26d6a
--- /dev/null
+++ b/web/src/features/experiments/constants/experimentsSearchRegistry.clienttest.ts
@@ -0,0 +1,37 @@
+// @vitest-environment node
+
+import { describe, expect, it } from "vitest";
+
+import { getExperimentsFilterConfig } from "@/src/features/experiments/components/table/filter-config";
+import {
+ EXPERIMENTS_FIELD_REGISTRY,
+ experimentsFieldRegistry,
+} from "./experimentsSearchRegistry";
+
+describe("experimentsFieldRegistry β€” scoped to the facets it renders", () => {
+ it("drops the dataset field when the page pins the dataset", () => {
+ // A dataset-scoped page omits the Dataset facet, and the sidebar strips any
+ // filter on a facet-less column on write β€” so offering `dataset:` there
+ // would take the token and silently discard it.
+ const scoped = experimentsFieldRegistry(
+ getExperimentsFilterConfig(["experimentDatasetId"]),
+ );
+
+ expect(scoped.resolveField("dataset")).toBeNull();
+ expect(scoped.resolveField("experimentDatasetName")).toBeNull();
+ expect(scoped.resolveField("name")).not.toBeNull();
+ });
+
+ it("stops advertising an example the scoped registry cannot resolve", () => {
+ const scoped = experimentsFieldRegistry(
+ getExperimentsFilterConfig(["experimentDatasetId"]),
+ );
+
+ expect(scoped.searchExamples).not.toContain("dataset:legal-answer-quality");
+ expect(scoped.searchExamples).toContain("name:sonnet");
+ // The unscoped registry is unchanged: it has the facet, so it keeps it.
+ expect(EXPERIMENTS_FIELD_REGISTRY.searchExamples).toContain(
+ "dataset:legal-answer-quality",
+ );
+ });
+});
diff --git a/web/src/features/experiments/constants/experimentsSearchRegistry.ts b/web/src/features/experiments/constants/experimentsSearchRegistry.ts
index 108f96c879..a6e38b8988 100644
--- a/web/src/features/experiments/constants/experimentsSearchRegistry.ts
+++ b/web/src/features/experiments/constants/experimentsSearchRegistry.ts
@@ -68,25 +68,57 @@ const EXPERIMENT_FIELD_OVERLAY = {
},
};
-export const EXPERIMENTS_FIELD_REGISTRY: FieldRegistry =
- fieldRegistryFromColumns(facetColumns(experimentsFilterConfig), {
+/** Authored per view, never derived from the field ids β€” see the search-bar README. */
+const SEARCH_EXAMPLES = [
+ "dataset:legal-answer-quality",
+ "name:sonnet",
+ "scores.groundedness:>0.8",
+];
+
+/**
+ * Built from the config the table actually renders, not the static one: a
+ * dataset-scoped page omits the Dataset facet, and the sidebar then strips any
+ * filter on that column on write. A registry that still offered `dataset:`
+ * there would accept the token and let it be dropped without a word.
+ */
+export const experimentsFieldRegistry = (
+ config: FilterConfig = experimentsFilterConfig,
+): FieldRegistry => {
+ const overlay = {
id: "experiments",
metadata: true,
- // See SCORE_COLUMN_IDS: `scores.` needs the canonical column names, which
- // the unification change introduces.
- scores: false,
+ // `scores.<name>` lowers onto the canonical `scores_avg` /
+ // `score_categories` / `score_booleans` columns, which is exactly what this
+ // view filters on since the score unification β€” so the bar can render a
+ // score filter the sidebar set, and parse one the user types. `traceScores.`
+ // stays closed: the level-agnostic columns already match at trace level, and
+ // the `trace_*` columns are no longer offered.
+ scores: true,
traceScores: false,
allowFreeText: false,
// `name` is the only text column, and it is what people look a run up by.
defaultTextField: "name",
recentSearches: true,
- // `dataset:` takes the dataset ID, not its name β€” the observed-value picker
- // offers the ids, so the examples must not imply a name works.
- searchExamples: [
- "dataset:legal-answer-quality",
- "name:sonnet",
- 'metadata."model":opus',
- ],
aiContextFields: AI_CONTEXT_FIELDS,
fields: EXPERIMENT_FIELD_OVERLAY,
+ } as const;
+
+ // The examples are authored, never generated β€” but a scoped config drops
+ // facets, and an example naming a dropped one would advertise a field the bar
+ // rejects. Built once to resolve them (aliases and `scores.` included), then
+ // rebuilt with the examples that survive.
+ const probe = fieldRegistryFromColumns(facetColumns(config), overlay);
+
+ return fieldRegistryFromColumns(facetColumns(config), {
+ ...overlay,
+ searchExamples: SEARCH_EXAMPLES.filter((example) => {
+ const separator = example.indexOf(":");
+ if (separator === -1) return true;
+ return probe.resolveField(example.slice(0, separator)) !== null;
+ }),
});
+};
+
+/** The unscoped registry: the server's AI-filter guard and the grammar tests. */
+export const EXPERIMENTS_FIELD_REGISTRY: FieldRegistry =
+ experimentsFieldRegistry();
diff --git a/web/src/features/experiments/fns/awaitsDatasetNames.clienttest.ts b/web/src/features/experiments/fns/awaitsDatasetNames.clienttest.ts
new file mode 100644
index 0000000000..6f2a37954c
--- /dev/null
+++ b/web/src/features/experiments/fns/awaitsDatasetNames.clienttest.ts
@@ -0,0 +1,38 @@
+import { describe, expect, it } from "vitest";
+import type { FilterState } from "@langfuse/shared";
+
+import { awaitsDatasetNames } from "./awaitsDatasetNames";
+
+const nameFilter: FilterState = [
+ {
+ column: "experimentDatasetName",
+ type: "stringOptions",
+ operator: "any of",
+ value: ["legal-answer-quality"],
+ },
+];
+
+describe("awaitsDatasetNames", () => {
+ it("waits while the names query is unsettled", () => {
+ expect(
+ awaitsDatasetNames(nameFilter, { isSuccess: false, isError: false }),
+ ).toBe(true);
+ });
+
+ it("lets the query through once the names settle, empty or failed", () => {
+ // Both states leave the name -> id map empty, which is why the gate cannot
+ // be keyed on the map: either one would hold the table loading forever.
+ expect(
+ awaitsDatasetNames(nameFilter, { isSuccess: true, isError: false }),
+ ).toBe(false);
+ expect(
+ awaitsDatasetNames(nameFilter, { isSuccess: false, isError: true }),
+ ).toBe(false);
+ });
+
+ it("never waits without a dataset-name filter", () => {
+ expect(awaitsDatasetNames([], { isSuccess: false, isError: false })).toBe(
+ false,
+ );
+ });
+});
diff --git a/web/src/features/experiments/fns/awaitsDatasetNames.ts b/web/src/features/experiments/fns/awaitsDatasetNames.ts
new file mode 100644
index 0000000000..b0636f495b
--- /dev/null
+++ b/web/src/features/experiments/fns/awaitsDatasetNames.ts
@@ -0,0 +1,19 @@
+import type { FilterState } from "@langfuse/shared";
+
+const DATASET_NAME_COLUMN = "experimentDatasetName";
+
+/**
+ * Whether a dataset-NAME filter still cannot be translated to an id, so the
+ * query has to wait. Keyed on the names query having SETTLED, never on the
+ * name -> id map being non-empty: a project with no datasets and a failed fetch
+ * both settle with an empty map, and gating on the DATA would hold the table in
+ * its loading skeleton forever. Once settled, an untranslatable name
+ * legitimately matches no rows.
+ */
+export function awaitsDatasetNames(
+ filters: FilterState,
+ datasetNames: { isSuccess: boolean; isError: boolean },
+): boolean {
+ if (datasetNames.isSuccess || datasetNames.isError) return false;
+ return filters.some((filter) => filter.column === DATASET_NAME_COLUMN);
+}
diff --git a/web/src/features/experiments/hooks/useExperimentFilterOptions.ts b/web/src/features/experiments/hooks/useExperimentFilterOptions.ts
index 8a0626be99..29e41f7c95 100644
--- a/web/src/features/experiments/hooks/useExperimentFilterOptions.ts
+++ b/web/src/features/experiments/hooks/useExperimentFilterOptions.ts
@@ -106,10 +106,19 @@ export function useExperimentFilterOptions({
};
}, [usedDatasets, filterOptions.data]);
+ // The SETTLE state of the names query, not its contents: callers that must
+ // wait for a name -> id translation cannot read that off the map, which is
+ // empty both while loading and for a project with no datasets.
+ const datasetNamesQuery = useMemo(
+ () => ({ isSuccess: datasets.isSuccess, isError: datasets.isError }),
+ [datasets.isSuccess, datasets.isError],
+ );
+
return {
filterOptions: experimentFilterOptions,
datasetIdByName,
datasetNameById,
+ datasetNamesQuery,
isFilterOptionsPending: datasets.isLoading || filterOptions.isLoading,
};
}
diff --git a/web/src/features/experiments/hooks/useExperimentsTableData.ts b/web/src/features/experiments/hooks/useExperimentsTableData.ts
index f026a0c647..9819eac2be 100644
--- a/web/src/features/experiments/hooks/useExperimentsTableData.ts
+++ b/web/src/features/experiments/hooks/useExperimentsTableData.ts
@@ -37,6 +37,13 @@ type UseExperimentsTableDataParams = {
page: number;
limit: number;
};
+ /**
+ * Hold the queries while a filter value still needs translating. A dataset
+ * NAME reaches the query as an id, so firing before the name -> id map lands
+ * would query a bogus id and show "No experiments" before flipping to the
+ * real rows.
+ */
+ enabled?: boolean;
orderByState: {
column: string;
order: "ASC" | "DESC";
@@ -48,6 +55,7 @@ export function useExperimentsTableData({
filterState,
paginationState,
orderByState,
+ enabled = true,
}: UseExperimentsTableDataParams) {
// Prepare query payloads
const getCountPayload = useMemo(
@@ -78,15 +86,17 @@ export function useExperimentsTableData({
// on a cold load and the rejected zod input surfaces as a "Bad Request" toast.
const isProjectReady = Boolean(projectId);
+ const canQuery = isProjectReady && enabled;
+
// Fetch experiments
const experimentsQuery = api.experiments.all.useQuery(getAllPayload, {
- enabled: isProjectReady,
+ enabled: canQuery,
refetchOnWindowFocus: true,
});
// Fetch total count
const totalCountQuery = api.experiments.countAll.useQuery(getCountPayload, {
- enabled: isProjectReady,
+ enabled: canQuery,
refetchOnWindowFocus: true,
});
@@ -99,7 +109,7 @@ export function useExperimentsTableData({
const mostRecentQuery = api.experiments.mostRecent.useQuery(
{ ...getCountPayload, limit: MOST_RECENT_FALLBACK_LIMIT },
{
- enabled: isProjectReady && isEmptyWindow,
+ enabled: canQuery && isEmptyWindow,
refetchOnWindowFocus: false,
},
);
@@ -132,7 +142,7 @@ export function useExperimentsTableData({
// Fetch metrics
const metricsQuery = api.experiments.metrics.useQuery(metricsPayload!, {
- enabled: isProjectReady && metricsPayload !== null,
+ enabled: canQuery && metricsPayload !== null,
refetchOnWindowFocus: false,
staleTime: 0,
});
@@ -140,9 +150,9 @@ export function useExperimentsTableData({
// Memoize joined data to prevent infinite re-renders
// Handle loading, error, and success states
const joinedData = useMemo(() => {
- // A disabled query is neither loading nor errored, so keep the table in its
- // loading state until the project id arrives instead of flashing "no rows".
- if (!isProjectReady || experimentsQuery.isLoading) {
+ // A disabled query is neither loading nor errored, so stay in the loading
+ // state while the gate is shut instead of flashing "no rows".
+ if (!canQuery || experimentsQuery.isLoading) {
return { status: "loading" as const, rows: undefined };
}
@@ -161,7 +171,7 @@ export function useExperimentsTableData({
metricsQuery.data,
);
}, [
- isProjectReady,
+ canQuery,
experimentsQuery.isLoading,
experimentsQuery.isError,
isEmptyWindow,
diff --git a/web/src/features/search-bar/lib/searchBarInvariants.clienttest.ts b/web/src/features/search-bar/lib/searchBarInvariants.clienttest.ts
index d4993020f5..4290ed1bda 100644
--- a/web/src/features/search-bar/lib/searchBarInvariants.clienttest.ts
+++ b/web/src/features/search-bar/lib/searchBarInvariants.clienttest.ts
@@ -763,9 +763,9 @@ const experimentsView: RegistryUnderTest = {
scoreContexts: [],
fieldValues: ["x", "sonnet", "5", "0.8", "a b"],
freeTextValues: ["hello", "run 12", "or", "!important"],
- // Scores stay in the sidebar on this view, and after the score unification
- // they sit on the SAME canonical columns the bar's `scores.` path recognizes β€”
- // so the serializer has to be told they are not the bar's to render.
+ // Score filters are the bar's to render on this view, so these must derive to
+ // valid text rather than being skipped. The `trace_*` columns are not, and
+ // stay sidebar-only.
sidebarFilters: [
[
{
@@ -846,14 +846,19 @@ describe("search bar invariants β€” experiments registry", () => {
).toBeNull();
});
- it("keeps score dot-paths closed until the columns are unified", () => {
- // The adapter lowers `scores.<name>` onto the canonical scores_avg /
- // score_categories / score_booleans columns. Experiments names its score
- // columns obs_* / trace_*, so an open `scores.` here would emit a filter on
- // a column this view does not have.
+ it("opens `scores.` and keeps `traceScores.` closed", () => {
+ // `scores.<name>` lowers onto the canonical scores_avg / score_categories /
+ // score_booleans columns, which is what this view filters on. Those columns
+ // already match at trace level, so a separate `traceScores.` namespace would
+ // offer a second way to say the same thing β€” against `trace_*` columns the
+ // sidebar no longer offers.
expect(
EXPERIMENTS_FIELD_REGISTRY.resolveField("scores.groundedness"),
- ).toBeNull();
+ ).toEqual({
+ type: "scores",
+ key: "groundedness",
+ level: "observation",
+ });
expect(EXPERIMENTS_FIELD_REGISTRY.resolveField("traceScores.x")).toBeNull();
// Metadata is unaffected β€” that column exists under its canonical name.
expect(EXPERIMENTS_FIELD_REGISTRY.resolveField("metadata.owner")).toEqual({
@@ -862,6 +867,28 @@ describe("search bar invariants β€” experiments registry", () => {
});
});
+ it("renders a score filter the sidebar set", () => {
+ // The sidebar and the bar edit ONE filter state, so a score filter added in
+ // the sidebar has to show up as a token β€” otherwise the bar reads as though
+ // no score filter were applied.
+ const derived = filterStateToQueryText(
+ [
+ {
+ type: "numberObject",
+ column: "scores_avg",
+ key: "citation_usable_rate",
+ operator: "=",
+ value: 4,
+ },
+ ],
+ {},
+ EXPERIMENTS_FIELD_REGISTRY,
+ );
+
+ expect(derived.text).toBe("scores.citation_usable_rate:4");
+ expect(derived.skippedFilters).toEqual([]);
+ });
+
it("rewrites a bare word onto the experiment name", () => {
const committed = planCommit(
"refund eval",