🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(experiments): add the search bar to the experiments table (#16756)
Nikita Kabardin9c13c2916089
diff --git a/web/src/__tests__/server/ai-filter-access.servertest.ts b/web/src/__tests__/server/ai-filter-access.servertest.tsindex d89b2c2f1f..342ab6f370 100644--- a/web/src/__tests__/server/ai-filter-access.servertest.ts+++ b/web/src/__tests__/server/ai-filter-access.servertest.ts@@ -122,6 +122,23 @@ describe("Ask AI filter generation access", () => {).not.toContain(NO_ACCESS);});+ it("rejects AI generation for a view with no prompt of its own", async () => {+ // The experiments bar deliberately has no AI prompt. Before the registry+ // lookup was total, this id fell through to the events registry, whose flag+ // is set — so the guard passed and answered with the events vocabulary.+ const { project, caller } = await prepare("OWNER");++ await expect(+ caller.searchBar.generateFilter({+ projectId: project.id,+ prompt: "runs where groundedness is low",+ registryId: "experiments",+ }),+ ).rejects.toMatchObject({+ message: "AI filter generation is not available for this view",+ });+ });+it("rejects a project role without project:read on both endpoints", async () => {const { project, caller } = await prepare("NONE");diff --git a/web/src/features/experiments/components/table/ExperimentsTable.tsx b/web/src/features/experiments/components/table/ExperimentsTable.tsxindex e1fe4ee153..461b85fd2a 100644--- a/web/src/features/experiments/components/table/ExperimentsTable.tsx+++ b/web/src/features/experiments/components/table/ExperimentsTable.tsx@@ -10,6 +10,11 @@ import { useEffect, useMemo, useState, useRef, useCallback } from "react";import { useQueryFilterState } from "@/src/features/filters/hooks/useFilterState";import { usePaginationState } from "@/src/hooks/usePaginationState";import { useSidebarFilterState } from "@/src/features/filters/hooks/useSidebarFilterState";+import { EXPERIMENTS_FIELD_REGISTRY } from "@/src/features/experiments/constants/experimentsSearchRegistry";+import { toObservedOptions } from "@/src/features/search-bar/lib/observed-options";+import { DEFAULT_SEARCH_TYPE } from "@/src/features/search-bar/lib/commit";+import { useEventsSearchBar } from "@/src/features/search-bar/hooks/useEventsSearchBar";+import { EventsSearchBarRow } from "@/src/features/search-bar/components/EventsSearchBarRow";import {getExperimentsFilterConfig,getExperimentsColumnName,@@ -371,6 +376,34 @@ export default function ExperimentsTable({[],);+ // Grammar search bar: an ADDITIONAL editor over the same FilterState the+ // facet sidebar edits. Score filtering stays in the sidebar here — see+ // experimentsSearchRegistry.+ const observedOptions = useMemo(+ () => toObservedOptions(filterOptions, isFilterOptionsPending),+ [filterOptions, isFilterOptionsPending],+ );+ // The experiments table has no full-text lane, so the registry rejects free+ // text and these stay inert.+ const noSearchLane = useCallback(() => {}, []);+ const {+ store: searchBarStore,+ commit: searchBarCommit,+ applyFilters: searchBarApplyFilters,+ } = useEventsSearchBar({+ projectId,+ tableName: filterConfig.tableName,+ enabled: true,+ filterState: queryFilter.explicitFilterState,+ searchQuery: null,+ searchType: DEFAULT_SEARCH_TYPE,+ observed: observedOptions,+ setFilterState: setFiltersWrapper,+ setSearchQuery: noSearchLane,+ setSearchType: noSearchLane,+ registry: EXPERIMENTS_FIELD_REGISTRY,+ });+const combinedFilterState = queryFilter.filterState.concat(dateRangeFilter,fixedFilter,@@ -776,37 +809,52 @@ export default function ExperimentsTable({setTimeRange={setTimeRange}/>)}- {/* Toolbar spanning full width */}- <DataTableToolbar- columns={columns}- filterState={queryFilter.filterState}- viewConfig={{- tableName: TableViewPresetTableName.Experiments,- projectId,- controllers: viewControllers,- }}- tableName={filterConfig.tableName}- isV4={true}- onColumnGroupToggle={handleColumnGroupToggle}- columnsWithCustomSelect={["name", "datasetId"]}- columnVisibility={columnVisibility}- setColumnVisibility={setColumnVisibilityState}- columnOrder={columnOrder}- setColumnOrder={setColumnOrder}- orderByState={orderByState}- rowHeight={rowHeight}- setRowHeight={setRowHeight}- timeRange={showControlsInPageHeader ? undefined : timeRange}- setTimeRange={showControlsInPageHeader ? undefined : setTimeRange}- actionButtons={[- <ExperimentsMultiSelectActionMenu- key="experiments-multi-select-actions"- projectId={projectId}- store={experimentsTableStore}- datasetIdByExperimentId={datasetIdByExperimentId}- />,- ]}- />+ {/* The composer and the toolbar stick together as one band so the+ toolbar cannot scroll under the composer and render half-clipped;+ pb-1.5 matches the other bar surfaces' spacing above the table. */}+ <div className="bg-background sticky top-0 z-30 pb-1.5">+ <EventsSearchBarRow+ projectId={projectId}+ tableName={filterConfig.tableName}+ store={searchBarStore}+ commit={searchBarCommit}+ observed={observedOptions}+ onApplyFilters={searchBarApplyFilters}+ registry={EXPERIMENTS_FIELD_REGISTRY}+ />+ {/* Toolbar spanning full width */}+ <DataTableToolbar+ rowClassName="my-1"+ columns={columns}+ filterState={queryFilter.filterState}+ viewConfig={{+ tableName: TableViewPresetTableName.Experiments,+ projectId,+ controllers: viewControllers,+ }}+ tableName={filterConfig.tableName}+ isV4={true}+ onColumnGroupToggle={handleColumnGroupToggle}+ columnsWithCustomSelect={["name", "datasetId"]}+ columnVisibility={columnVisibility}+ setColumnVisibility={setColumnVisibilityState}+ columnOrder={columnOrder}+ setColumnOrder={setColumnOrder}+ orderByState={orderByState}+ rowHeight={rowHeight}+ setRowHeight={setRowHeight}+ timeRange={showControlsInPageHeader ? undefined : timeRange}+ setTimeRange={showControlsInPageHeader ? undefined : setTimeRange}+ actionButtons={[+ <ExperimentsMultiSelectActionMenu+ key="experiments-multi-select-actions"+ projectId={projectId}+ store={experimentsTableStore}+ datasetIdByExperimentId={datasetIdByExperimentId}+ />,+ ]}+ />+ </div>{/* Charts section - Collapsible Accordion */}{tableDateRange && (diff --git a/web/src/features/experiments/components/table/filter-config.ts b/web/src/features/experiments/components/table/filter-config.tsindex cebe618b11..bb005a05b1 100644--- a/web/src/features/experiments/components/table/filter-config.ts+++ b/web/src/features/experiments/components/table/filter-config.ts@@ -136,7 +136,7 @@ export const getExperimentsColumnName = (id: string): string => {return column.name;};-const experimentsFilterConfig: FilterConfig = {+export const experimentsFilterConfig: FilterConfig = {tableName: "experiments",columnDefinitions: experimentsTableCols,diff --git a/web/src/features/experiments/constants/experimentsSearchRegistry.ts b/web/src/features/experiments/constants/experimentsSearchRegistry.tsnew file mode 100644index 0000000000..a6c556873c--- /dev/null+++ b/web/src/features/experiments/constants/experimentsSearchRegistry.ts@@ -0,0 +1,87 @@+import {+ fieldRegistryFromColumns,+ type FieldRegistry,+} from "@/src/features/search-bar/lib/fields";+import type { FilterConfig } from "@/src/features/filters/lib/filter-config";++import { experimentsFilterConfig } from "@/src/features/experiments/components/table/filter-config";++/**+ * Derived from the sidebar's FACET list, not the raw `ColumnDefinition[]` — the+ * facets are the view's curated filter surface, so the bar stays a strict subset+ * of the sidebar and cannot author a filter the sidebar is unable to show or+ * clear. See `features/search-bar/README.md` ("Extending to other views").+ */+function facetColumns(config: FilterConfig) {+ const exposed = new Set(config.facets.map((facet) => facet.column));+ return config.columnDefinitions.filter(+ (column) => exposed.has(column.id) && !EXCLUDED_COLUMN_IDS.has(column.id),+ );+}++const AI_CONTEXT_FIELDS = [+ { observedOptionsKey: "name", promptLabel: "name" },+] as const;++/**+ * Score filtering stays in the sidebar on this surface, deliberately.+ *+ * The grammar lowers `scores.<name>` onto the canonical `scores_avg` /+ * `score_categories` / `score_booleans` columns (`SCORE_COLUMNS`, applied+ * unconditionally by the adapter). Experiments names its score columns+ * `obs_*` / `trace_*`, so a `scores.` token here would emit a filter on a+ * column this view does not have. Renaming those columns is exactly what the+ * score-unification change does, so `scores.` arrives with it rather than+ * shipping broken now. These columns are dropped from the derived field list+ * too: `obs_score_categories` is `categoryOptions`, not `*Object`, so the+ * helper's keyed-score exclusion does not catch it and it would otherwise+ * surface as a bogus keyless field.+ */+const EXCLUDED_COLUMN_IDS: ReadonlySet<string> = new Set([+ // `experimentDatasetId` holds the dataset ID while the sidebar's picker shows+ // the dataset NAME (`displayValue`). A text query bar can only show what the+ // query text contains, so offering it here would put `dataset:cmtc1z…` in a+ // pill next to a sidebar that says `legal-answer-quality`. Making the bar+ // speak names would require the derived committed text to depend on+ // async-loaded options, which breaks the one invariant every bar surface+ // relies on. Dataset filtering stays in the sidebar.+ "experimentDatasetId",+ "obs_scores_avg",+ "obs_score_categories",+ "obs_score_booleans",+ "trace_scores_avg",+ "trace_score_categories",+ "trace_score_booleans",+]);++/**+ * Grammar overlay: aliases, units and prose that `ColumnDefinition` does not+ * carry. Units matter because every run metric here is a magnitude someone+ * types with an implied unit (`cost:>1`, `latency:>30`).+ */+const EXPERIMENT_FIELD_OVERLAY = {+ name: {+ aliases: ["experimentname", "experiment_name", "experiment"],+ label: "Experiment name",+ description: "Experiment name",+ },+};++export const EXPERIMENTS_FIELD_REGISTRY: FieldRegistry =+ fieldRegistryFromColumns(facetColumns(experimentsFilterConfig), {+ id: "experiments",+ metadata: true,+ // See SCORE_COLUMN_IDS: `scores.` needs the canonical column names, which+ // the unification change introduces.+ scores: false,+ traceScores: false,+ allowFreeText: false,+ // `name` is the only text column, and it is what people look a run up by.+ defaultTextField: "name",+ recentSearches: true,+ // `dataset:` takes the dataset ID, not its name — the observed-value picker+ // offers the ids, so the examples must not imply a name works.+ searchExamples: ["name:sonnet", "-name:baseline", 'metadata."model":opus'],+ aiContextFields: AI_CONTEXT_FIELDS,+ fields: EXPERIMENT_FIELD_OVERLAY,+ });diff --git a/web/src/features/search-bar/lib/fields.ts b/web/src/features/search-bar/lib/fields.tsindex 538429272d..ec8ba49ea6 100644--- a/web/src/features/search-bar/lib/fields.ts+++ b/web/src/features/search-bar/lib/fields.ts@@ -73,7 +73,8 @@ export type FieldRegistry = {| "evaluationRules"| "evaluatorSamples"| "ruleSamples"- | "sessions";+ | "sessions"+ | "experiments";fields: readonly FieldDef[];columns: readonly ColumnDefinition[];allowFreeText: boolean;diff --git a/web/src/features/search-bar/lib/searchBarInvariants.clienttest.ts b/web/src/features/search-bar/lib/searchBarInvariants.clienttest.tsindex 290987e1ae..b01b3c4b87 100644--- a/web/src/features/search-bar/lib/searchBarInvariants.clienttest.ts+++ b/web/src/features/search-bar/lib/searchBarInvariants.clienttest.ts@@ -13,6 +13,7 @@ import {RULE_SAMPLE_FIELD_REGISTRY,} from "@/src/features/evals/v2/constants/evaluatorSearchRegistry";import { SESSIONS_FIELD_REGISTRY } from "@/src/features/filters/config/sessionsSearchRegistry";+import { EXPERIMENTS_FIELD_REGISTRY } from "@/src/features/experiments/constants/experimentsSearchRegistry";import { validateQuery } from "./validate";import { planCommit } from "./commit";import { filterStateToQueryText } from "./filter-state-to-query";@@ -659,3 +660,85 @@ describe("search bar invariants — sessions registry", () => {expect(RULE_FIELD_REGISTRY.recentSearches).toBe(false);});});++const experimentsView: RegistryUnderTest = {+ name: "experiments",+ registry: EXPERIMENTS_FIELD_REGISTRY,+ extraKeys: ["metadata.owner", 'metadata."my key"'],+ scoreContexts: [],+ fieldValues: ["x", "sonnet", "5", "0.8", "a b"],+ freeTextValues: ["hello", "run 12", "or", "!important"],+};++describe("search bar invariants — experiments registry", () => {+ it("holds all three invariants (parity, round-trip, serialize symmetry)", () => {+ const failures = runSearchBarInvariants(experimentsView);+ expect(+ failures,+ failures.length === 0+ ? "ok"+ : `\n${failures+ .slice(0, 25)+ .map((f) => ` [${f.invariant}] ${f.case} — ${f.detail}`)+ .join(+ "\n",+ )}${failures.length > 25 ? `\n …and ${failures.length - 25} more` : ""}`,+ ).toEqual([]);+ });++ it("exposes only the sidebar facets whose columns are actually filterable", () => {+ expect(EXPERIMENTS_FIELD_REGISTRY.fields.map((f) => f.id).sort()).toEqual([+ "name",+ ]);+ // The dataset column holds an ID while the sidebar shows the name, so the+ // bar would contradict it — see the registry's EXCLUDED_COLUMN_IDS note.+ expect(EXPERIMENTS_FIELD_REGISTRY.resolveField("dataset")).toBeNull();+ });++ it("keeps score dot-paths closed until the columns are unified", () => {+ // The adapter lowers `scores.<name>` onto the canonical scores_avg /+ // score_categories / score_booleans columns. Experiments names its score+ // columns obs_* / trace_*, so an open `scores.` here would emit a filter on+ // a column this view does not have.+ expect(+ EXPERIMENTS_FIELD_REGISTRY.resolveField("scores.groundedness"),+ ).toBeNull();+ expect(EXPERIMENTS_FIELD_REGISTRY.resolveField("traceScores.x")).toBeNull();+ // Metadata is unaffected — that column exists under its canonical name.+ expect(EXPERIMENTS_FIELD_REGISTRY.resolveField("metadata.owner")).toEqual({+ type: "metadata",+ key: "owner",+ });+ });++ it("rewrites a bare word onto the experiment name", () => {+ const committed = planCommit(+ "refund eval",+ undefined,+ EXPERIMENTS_FIELD_REGISTRY,+ );+ expect(committed).toMatchObject({+ status: "committed",+ searchQuery: null,+ filters: [+ {+ column: "name",+ type: "string",+ operator: "contains",+ value: "refund eval",+ },+ ],+ });+ });++ it("keeps the run metrics out — the backend silently drops them", () => {+ // itemCount / errorCount / totalCost / latencyAvg have ColumnDefinitions for+ // display and sorting but no UiColumnMapping, and the repository partitions+ // filters by allow-list, discarding anything in neither group. A field here+ // would render a pill and change nothing.+ for (const id of ["itemCount", "errorCount", "totalCost", "latencyAvg"]) {+ expect(EXPERIMENTS_FIELD_REGISTRY.resolveField(id)).toBeNull();+ }+ expect(EXPERIMENTS_FIELD_REGISTRY.resolveField("cost")).toBeNull();+ });+});diff --git a/web/src/features/search-bar/server/router.ts b/web/src/features/search-bar/server/router.tsindex 1702075a9e..24e83c6003 100644--- a/web/src/features/search-bar/server/router.ts+++ b/web/src/features/search-bar/server/router.ts@@ -55,7 +55,12 @@ import {} from "@/src/features/evals/v2/constants/evaluatorSearchRegistry";import { DATASET_NAME_COLUMN } from "@/src/features/evals/v2/utils/datasetNameFilter";import { SESSIONS_FIELD_REGISTRY } from "@/src/features/filters/config/sessionsSearchRegistry";-import { EVENTS_FIELD_REGISTRY, withFieldOptions } from "../lib/fields";+import { EXPERIMENTS_FIELD_REGISTRY } from "@/src/features/experiments/constants/experimentsSearchRegistry";+import {+ EVENTS_FIELD_REGISTRY,+ withFieldOptions,+ type FieldRegistry,+} from "../lib/fields";// Caps shared with `observedScoreNamesFromOptions` (the client-side builder),// which sends a set as undefined instead of ever exceeding them.@@ -63,18 +68,34 @@ const scoreNameList = z.array(z.string().max(MAX_SCORE_NAME_LENGTH)).max(MAX_SCORE_NAMES_PER_TYPE);+const REGISTRY_IDS = [+ "events",+ "evaluationRules",+ "evaluatorSamples",+ "ruleSamples",+ "sessions",+ "experiments",+] as const;++/**+ * Every accepted registryId maps to its registry EXPLICITLY. A fallback here is+ * what made an unmapped view generate against the events vocabulary while the+ * `aiFilterPrompt` guard below read the events flag and let it through; with a+ * total map, adding an id without a registry is a type error instead.+ */+const REGISTRIES_BY_ID: Record<(typeof REGISTRY_IDS)[number], FieldRegistry> = {+ events: EVENTS_FIELD_REGISTRY,+ evaluationRules: RULE_FIELD_REGISTRY,+ evaluatorSamples: EVALUATOR_FIELD_REGISTRY,+ ruleSamples: RULE_SAMPLE_FIELD_REGISTRY,+ sessions: SESSIONS_FIELD_REGISTRY,+ experiments: EXPERIMENTS_FIELD_REGISTRY,+};+const GenerateFilterInput = z.object({projectId: z.string(),prompt: z.string().min(1).max(2048),- registryId: z- .enum([- "events",- "evaluationRules",- "evaluatorSamples",- "ruleSamples",- "sessions",- ])- .default("events"),+ registryId: z.enum(REGISTRY_IDS).default("events"),/** Existing bar query text, so the model refines the current filters. */currentQuery: z.string().max(4096).optional(),/** Project data context (observed values, metadata keys, result count) built@@ -100,19 +121,10 @@ export const searchBarRouter = createTRPCRouter({.input(GenerateFilterInput).mutation(async ({ input, ctx }) => {try {- const baseRegistry =- input.registryId === "evaluationRules"- ? RULE_FIELD_REGISTRY- : input.registryId === "evaluatorSamples"- ? EVALUATOR_FIELD_REGISTRY- : input.registryId === "ruleSamples"- ? RULE_SAMPLE_FIELD_REGISTRY- : input.registryId === "sessions"- ? SESSIONS_FIELD_REGISTRY- : EVENTS_FIELD_REGISTRY;- // Defence in depth for the client gate above: generating against the- // fallback (events) prompt for a view that has no branch of its own- // produces filters for columns that view does not have.+ const baseRegistry = REGISTRIES_BY_ID[input.registryId];+ // Defence in depth for the client gate above: a view that has no AI+ // prompt of its own must be rejected, not answered with another view's+ // column vocabulary.if (!baseRegistry.aiFilterPrompt) {throw new TRPCError({code: "BAD_REQUEST",