🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(evals): filter evaluator samples by dataset name (#16799)
Tobias Wochingerb5a9ed2ebf63
diff --git a/web/src/__tests__/server/ai-filter-access.servertest.ts b/web/src/__tests__/server/ai-filter-access.servertest.tsindex ec2b4d128e..d89b2c2f1f 100644--- a/web/src/__tests__/server/ai-filter-access.servertest.ts+++ b/web/src/__tests__/server/ai-filter-access.servertest.ts@@ -187,7 +187,44 @@ describe("Ask AI filter generation access", () => {prompt: "traces from today",}),).resolves.toMatchObject({ filters: [] });- expect(llmMocks.generateLLMText).toHaveBeenCalledOnce();++ await prisma.dataset.create({+ data: {+ id: "ai-filter-dataset",+ projectId: project.id,+ name: "Filter QA Dataset",+ },+ });+ llmMocks.generateLLMText.mockResolvedValue({+ text: JSON.stringify([+ {+ type: "stringOptions",+ column: "datasetName",+ operator: "any of",+ value: ["Filter QA Dataset"],+ },+ ]),+ });++ for (const registryId of ["evaluatorSamples", "ruleSamples"] as const) {+ await expect(+ caller.searchBar.generateFilter({+ projectId: project.id,+ prompt: "only the Filter QA Dataset",+ registryId,+ }),+ ).resolves.toMatchObject({+ filters: [+ {+ type: "stringOptions",+ column: "experimentDatasetId",+ operator: "any of",+ value: ["ai-filter-dataset"],+ },+ ],+ });+ }+ expect(llmMocks.generateLLMText).toHaveBeenCalledTimes(3);} finally {(env as { NEXT_PUBLIC_LANGFUSE_CLOUD_REGION?: string }diff --git a/web/src/__tests__/server/unit/searchBarFilterPrompt.servertest.ts b/web/src/__tests__/server/unit/searchBarFilterPrompt.servertest.tsindex ef064279b2..d6328e2aaf 100644--- a/web/src/__tests__/server/unit/searchBarFilterPrompt.servertest.ts+++ b/web/src/__tests__/server/unit/searchBarFilterPrompt.servertest.ts@@ -5,6 +5,7 @@ import {FIELDS,SCORE_COLUMNS,type FieldDef,+ withFieldOptions,} from "@/src/features/search-bar/lib/fields";import {buildFilterContextMessage,@@ -13,6 +14,10 @@ import {} from "@/src/features/search-bar/server/buildFilterPrompt";import { filterStateToQueryText } from "@/src/features/search-bar/lib/filter-state-to-query";import { RULE_FIELD_REGISTRY } from "@/src/features/evals/v2/constants/ruleSearchRegistry";+import {+ EVALUATOR_FIELD_REGISTRY,+ RULE_SAMPLE_FIELD_REGISTRY,+} from "@/src/features/evals/v2/constants/evaluatorSearchRegistry";import { parseGeneratedFilters } from "@/src/features/search-bar/server/parseFilterCompletion";// A representative `singleFilter` for the FilterState `type` the prompt tells@@ -139,6 +144,61 @@ describe("evaluation rule AI registry", () => {});});+describe("sample-filter AI registries", () => {+ const options = [{ value: "dataset-id", displayValue: "Filter QA Dataset" }];++ it("uses rule-specific prompt semantics for rule samples", () => {+ const prompt = buildFilterSystemPrompt(+ "Monday, 2026-06-15T00:00:00.000Z",+ RULE_SAMPLE_FIELD_REGISTRY,+ );++ expect(prompt).toContain(+ "observation filters for a Langfuse evaluation rule",+ );+ expect(prompt).toContain("- datasetName");+ expect(prompt).not.toContain("observability (v4 events) table");+ });++ it("projects known names to IDs and drops unknown AI values", () => {+ const registry = withFieldOptions(+ EVALUATOR_FIELD_REGISTRY,+ "datasetName",+ options,+ );+ const completion = (value: string) =>+ JSON.stringify([+ {+ type: "stringOptions",+ column: "datasetName",+ operator: "any of",+ value: [value],+ },+ ]);++ expect(+ parseGeneratedFilters(+ completion("Filter QA Dataset"),+ undefined,+ registry,+ ),+ ).toMatchObject({+ filters: [+ {+ type: "stringOptions",+ column: "experimentDatasetId",+ operator: "any of",+ value: ["dataset-id"],+ },+ ],+ droppedCount: 0,+ });+ expect(+ parseGeneratedFilters(completion("Missing dataset"), undefined, registry),+ ).toMatchObject({ filters: [], droppedCount: 1 });+ });+});+describe("buildFilterContextMessage", () => {it("returns null when neither current query nor data context is given", () => {expect(buildFilterContextMessage()).toBeNull();diff --git a/web/src/features/evals/hooks/useEvalConfigFilterOptions.clienttest.ts b/web/src/features/evals/hooks/useEvalConfigFilterOptions.clienttest.tsindex e57961ad39..84e0dc9e9f 100644--- a/web/src/features/evals/hooks/useEvalConfigFilterOptions.clienttest.ts+++ b/web/src/features/evals/hooks/useEvalConfigFilterOptions.clienttest.ts@@ -6,6 +6,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest";import { useEvalConfigFilterOptions } from "./useEvalConfigFilterOptions";const mocks = vi.hoisted(() => ({+ datasetsData: [] as Array<{ id: string; name: string }>,eventsData: {} as Record<string, unknown>,eventsInput: undefined as unknown,eventsOptions: undefined as unknown,@@ -19,7 +20,7 @@ vi.mock("@/src/utils/api", () => ({api: {datasets: {allDatasetMeta: {- useQuery: () => ({ data: [] }),+ useQuery: () => ({ data: mocks.datasetsData }),},},events: {@@ -57,6 +58,7 @@ vi.mock("@/src/utils/api", () => ({describe("useEvalConfigFilterOptions", () => {beforeEach(() => {+ mocks.datasetsData = [];mocks.eventsData = {};mocks.eventsInput = undefined;mocks.eventsOptions = undefined;@@ -136,4 +138,25 @@ describe("useEvalConfigFilterOptions", () => {calledToolNames: [{ value: "legacy-tool" }],});});++ it("shows dataset names while keeping IDs in legacy evaluator filters", () => {+ mocks.datasetsData = [{ id: "dataset-id", name: "Filter QA Dataset" }];++ const { result } = renderHook(() =>+ useEvalConfigFilterOptions({+ projectId: "project-id",+ useEventsTable: false,+ includeLegacyTraceOptions: false,+ }),+ );++ expect(result.current.experimentEvalFilterOptions).toEqual({+ experimentDatasetId: [+ {+ value: "dataset-id",+ displayValue: "Filter QA Dataset",+ },+ ],+ });+ });});diff --git a/web/src/features/evals/v2/components/Evaluators/Testing/components/EvaluatorSampleObservationSelector/EvaluatorSampleObservationSelector.tsx b/web/src/features/evals/v2/components/Evaluators/Testing/components/EvaluatorSampleObservationSelector/EvaluatorSampleObservationSelector.tsxindex 64b6514c91..9c06bb125e 100644--- a/web/src/features/evals/v2/components/Evaluators/Testing/components/EvaluatorSampleObservationSelector/EvaluatorSampleObservationSelector.tsx+++ b/web/src/features/evals/v2/components/Evaluators/Testing/components/EvaluatorSampleObservationSelector/EvaluatorSampleObservationSelector.tsx@@ -10,6 +10,7 @@ import {SampleObservationSelectorBase,type SampleObservation,} from "@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/SampleObservationSelectorBase";+import { EVALUATOR_FIELD_REGISTRY } from "@/src/features/evals/v2/constants/evaluatorSearchRegistry";const preserveObservedOptions: Parameters<typeof SampleObservationSelectorBase@@ -85,7 +86,7 @@ export function EvaluatorSampleObservationSelector({onFilterStateChange={onFilterStateChange}previewFilters={filterState}tableName="evaluator-sample-observations"- registry={undefined}+ registry={EVALUATOR_FIELD_REGISTRY}selectedObservationId={selectedObservationId}onSelect={onSelect}onOpenTrace={onOpenTrace}diff --git a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/SampleObservationSelectorBase.tsx b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/SampleObservationSelectorBase.tsxindex 6b5952f32f..2d4d1ddde1 100644--- a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/SampleObservationSelectorBase.tsx+++ b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/SampleObservationSelectorBase.tsx@@ -13,6 +13,7 @@ import { DataTableColumnVisibilityFilter } from "@/src/components/table/data-tabimport { useRowHeightLocalStorage } from "@/src/components/table/data-table-row-height-switch";import type { LangfuseColumnDef } from "@/src/components/table/types";import { Button } from "@/src/components/ui/button";+import { Skeleton } from "@/src/components/ui/skeleton";import { useEventsFilterOptions } from "@/src/features/events/hooks/useEventsFilterOptions";import { EventsSearchBarRow } from "@/src/features/search-bar/components/EventsSearchBarRow";import { useEventsSearchBar } from "@/src/features/search-bar/hooks/useEventsSearchBar";@@ -20,6 +21,7 @@ import { buildAiContext } from "@/src/features/search-bar/lib/ai-context";import {type FieldRegistry,EVENTS_FIELD_REGISTRY,+ withFieldOptions,} from "@/src/features/search-bar/lib/fields";import {observedScoreNamesFromOptions,@@ -33,6 +35,10 @@ import { SectionHeader } from "@/src/features/evals/v2/components/Evaluators/Tesimport { EXPERIMENTS_AND_EVALS_EXCLUSION_FILTERS } from "@/src/features/evals/v2/constants/experimentAndEvalFilters";import { dedupeObservationPages } from "@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/dedupeObservations";import { toggleExampleFilters } from "@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/toggleExampleFilters";+import {+ DATASET_NAME_COLUMN,+ addDatasetNameObservedOptions,+} from "@/src/features/evals/v2/utils/datasetNameFilter";import { useReusableRuleFilterPresets } from "@/src/features/evals/v2/hooks/useReusableRuleFilterPresets";import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";@@ -171,9 +177,31 @@ export function SampleObservationSelectorBase(mapObservedOptions,} = props;const activeRegistry = registry ?? EVENTS_FIELD_REGISTRY;+ const datasets = api.datasets.allDatasetMeta.useQuery(+ { projectId },+ {+ refetchOnMount: false,+ refetchOnWindowFocus: false,+ refetchOnReconnect: false,+ staleTime: Infinity,+ },+ );+ const datasetOptions = useMemo(() => datasets.data ?? [], [datasets.data]);+ const searchRegistry = useMemo(+ () =>+ withFieldOptions(+ activeRegistry,+ DATASET_NAME_COLUMN,+ datasetOptions.map((dataset) => ({+ value: dataset.id,+ displayValue: dataset.name,+ })),+ ),+ [activeRegistry, datasetOptions],+ );const reusableRuleFilters = useReusableRuleFilterPresets(projectId,- activeRegistry,+ searchRegistry,);const capture = usePostHogClientCapture();const onQueryPresetPick = useCallback(@@ -222,20 +250,21 @@ export function SampleObservationSelectorBase(includeApproxCount: true,lazy: true,});- const observed = useMemo(- () =>- mapObservedOptions(- toObservedOptions(- options.filterOptions,- options.isFilterOptionsPending,- ),- ),- [mapObservedOptions, options.filterOptions, options.isFilterOptionsPending],- );+ const observed = useMemo(() => {+ const mapped = mapObservedOptions(+ toObservedOptions(options.filterOptions, options.isFilterOptionsPending),+ );+ return addDatasetNameObservedOptions(mapped, datasetOptions);+ }, [+ datasetOptions,+ mapObservedOptions,+ options.filterOptions,+ options.isFilterOptionsPending,+ ]);const search = useEventsSearchBar({projectId,tableName,- enabled: true,+ enabled: !datasets.isPending,filterState,searchQuery,searchType,@@ -249,7 +278,7 @@ export function SampleObservationSelectorBase(setSearchType: (next) => {setSearchType(next);},- ...(registry ? { registry } : {}),+ registry: searchRegistry,});// listCursor always reads in the events table's stable start_time DESC tuple// order, so it takes no orderBy.@@ -329,10 +358,10 @@ export function SampleObservationSelectorBase(resultCount: observationQuery.isSuccess? matchingObservations.length: null,- registry: activeRegistry,+ registry: searchRegistry,}),[- activeRegistry,+ searchRegistry,matchingObservations,observationQuery.isSuccess,observed,@@ -340,10 +369,10 @@ export function SampleObservationSelectorBase();const aiScoreNames = useMemo(() =>- activeRegistry.scores+ searchRegistry.scores? observedScoreNamesFromOptions(observed): undefined,- [activeRegistry, observed],+ [searchRegistry, observed],);const selectionToReconcile = observationQuery.isSuccess? resolveSelection(matchingObservations, selectedObservationId)@@ -465,22 +494,26 @@ export function SampleObservationSelectorBase(tooltip={filterTooltip}trailing={null}/>- <EventsSearchBarRow- projectId={projectId}- tableName={tableName}- store={search.store}- commit={search.commit}- observed={observed}- erroredColumns={options.erroredColumns}- {...(registry ? { registry } : {})}- onApplyFilters={search.applyFilters}- onRequestColumns={options.requestColumns}- presetSections={reusableRuleFilters.sections}- onQueryPresetPick={onQueryPresetPick}- aiDataContext={aiDataContext}- aiScoreNames={aiScoreNames}- className="p-0"- />+ {datasets.isPending ? (+ <Skeleton className="h-10 w-full" />+ ) : (+ <EventsSearchBarRow+ projectId={projectId}+ tableName={tableName}+ store={search.store}+ commit={search.commit}+ observed={observed}+ erroredColumns={options.erroredColumns}+ registry={searchRegistry}+ onApplyFilters={search.applyFilters}+ onRequestColumns={options.requestColumns}+ presetSections={reusableRuleFilters.sections}+ onQueryPresetPick={onQueryPresetPick}+ aiDataContext={aiDataContext}+ aiScoreNames={aiScoreNames}+ className="p-0"+ />+ )}<div className="flex flex-wrap gap-2">{EXAMPLES.map((example) => (<Buttondiff --git a/web/src/features/evals/v2/components/Rules/RuleSetup/components/RuleFilterStep.tsx b/web/src/features/evals/v2/components/Rules/RuleSetup/components/RuleFilterStep.tsxindex 9de2ba67db..808412e136 100644--- a/web/src/features/evals/v2/components/Rules/RuleSetup/components/RuleFilterStep.tsx+++ b/web/src/features/evals/v2/components/Rules/RuleSetup/components/RuleFilterStep.tsx@@ -4,7 +4,7 @@ import { useStore } from "zustand";import { RuleSampleObservationSelector } from "@/src/features/evals/v2/components/Evaluators/Testing/components/RuleSampleObservationSelector/RuleSampleObservationSelector";import { Stepper } from "@/src/features/evals/v2/components/Stepper/Stepper";import type { RuleSetupStore } from "@/src/features/evals/v2/types/rules";-import { RULE_FIELD_REGISTRY } from "@/src/features/evals/v2/constants/ruleSearchRegistry";+import { RULE_SAMPLE_FIELD_REGISTRY } from "@/src/features/evals/v2/constants/evaluatorSearchRegistry";import { env } from "@/src/env.mjs";import { RuleSamplingSection } from "./RuleSamplingSection";@@ -38,7 +38,7 @@ export function RuleFilterStep({filterState={filter}onFilterStateChange={actions.setFilter}tableName="evaluation-rule-matching-observations"- registry={RULE_FIELD_REGISTRY}+ registry={RULE_SAMPLE_FIELD_REGISTRY}selectedObservationId={selectedObservationId}onSelect={actions.setSelectedObservation}onOpenTrace={(observation) => {diff --git a/web/src/features/evals/v2/constants/evaluatorSearchRegistry.ts b/web/src/features/evals/v2/constants/evaluatorSearchRegistry.tsnew file mode 100644index 0000000000..222623a0f9--- /dev/null+++ b/web/src/features/evals/v2/constants/evaluatorSearchRegistry.ts@@ -0,0 +1,39 @@+import { DATASET_NAME_FILTER_COLUMN } from "@/src/features/evals/v2/utils/datasetNameFilter";+import { RULE_FIELD_REGISTRY } from "@/src/features/evals/v2/constants/ruleSearchRegistry";+import {+ EVENTS_FIELD_REGISTRY,+ extendFieldRegistryWithColumns,+ type FieldRegistry,+} from "@/src/features/search-bar/lib/fields";++const withDatasetNameAiContext = (+ registry: FieldRegistry,+): FieldRegistry["aiContextFields"] =>+ registry.aiContextFields+ .filter(+ ({ observedOptionsKey }) => observedOptionsKey !== "experimentDatasetId",+ )+ .concat({+ observedOptionsKey: "datasetName",+ promptLabel: "datasetName",+ });++export const EVALUATOR_FIELD_REGISTRY = {+ ...extendFieldRegistryWithColumns(+ EVENTS_FIELD_REGISTRY,+ [DATASET_NAME_FILTER_COLUMN],+ { datasetName: { filterColumn: "experimentDatasetId" } },+ ),+ id: "evaluatorSamples" as const,+ aiContextFields: withDatasetNameAiContext(EVENTS_FIELD_REGISTRY),+};++export const RULE_SAMPLE_FIELD_REGISTRY = {+ ...extendFieldRegistryWithColumns(+ RULE_FIELD_REGISTRY,+ [DATASET_NAME_FILTER_COLUMN],+ { datasetName: { filterColumn: "experimentDatasetId" } },+ ),+ id: "ruleSamples" as const,+ aiContextFields: withDatasetNameAiContext(RULE_FIELD_REGISTRY),+};diff --git a/web/src/features/evals/v2/hooks/useReusableRuleFilterPresets.clienttest.ts b/web/src/features/evals/v2/hooks/useReusableRuleFilterPresets.clienttest.tsindex 35dd9c2072..fb550b7a77 100644--- a/web/src/features/evals/v2/hooks/useReusableRuleFilterPresets.clienttest.ts+++ b/web/src/features/evals/v2/hooks/useReusableRuleFilterPresets.clienttest.ts@@ -4,8 +4,12 @@ import { describe, expect, it } from "vitest";import type { FilterState } from "@langfuse/shared";import { RULE_FIELD_REGISTRY } from "@/src/features/evals/v2/constants/ruleSearchRegistry";+import { RULE_SAMPLE_FIELD_REGISTRY } from "@/src/features/evals/v2/constants/evaluatorSearchRegistry";import { prepareReusableRuleFilterPresets } from "@/src/features/evals/v2/hooks/useReusableRuleFilterPresets";-import { EVENTS_FIELD_REGISTRY } from "@/src/features/search-bar/lib/fields";+import {+ EVENTS_FIELD_REGISTRY,+ withFieldOptions,+} from "@/src/features/search-bar/lib/fields";const reusableFilters = [{@@ -87,4 +91,32 @@ describe("prepareReusableRuleFilterPresets", () => {expect(events.sections[0]?.options[0]?.query).toBe("name:=checkout");expect(rules.sections[0]?.options[0]?.query).toBe("name:checkout");});++ it("renders dataset-scoped presets with names instead of IDs", () => {+ const datasetFilters = [+ {+ latestRuleId: "rule-dataset",+ filter: [+ {+ column: "experimentDatasetId",+ type: "stringOptions",+ operator: "any of",+ value: ["dataset-id"],+ },+ ] satisfies FilterState,+ evaluatorCount: 1,+ updatedAt: new Date("2026-02-01T00:00:00.000Z"),+ },+ ];+ const registry = withFieldOptions(+ RULE_SAMPLE_FIELD_REGISTRY,+ "datasetName",+ [{ value: "dataset-id", displayValue: "Filter QA Dataset" }],+ );++ expect(+ prepareReusableRuleFilterPresets(datasetFilters, registry).sections[0]+ ?.options[0]?.query,+ ).toBe('datasetName:"Filter QA Dataset"');+ });});diff --git a/web/src/features/evals/v2/utils/datasetNameFilter.clienttest.ts b/web/src/features/evals/v2/utils/datasetNameFilter.clienttest.tsnew file mode 100644index 0000000000..50519820d5--- /dev/null+++ b/web/src/features/evals/v2/utils/datasetNameFilter.clienttest.ts@@ -0,0 +1,217 @@+import type { FilterState } from "@langfuse/shared";+import { describe, expect, it } from "vitest";++import {+ EVALUATOR_FIELD_REGISTRY,+ RULE_SAMPLE_FIELD_REGISTRY,+} from "@/src/features/evals/v2/constants/evaluatorSearchRegistry";+import { planCommit } from "@/src/features/search-bar/lib/commit";+import {+ type FieldRegistry,+ withFieldOptions,+} from "@/src/features/search-bar/lib/fields";+import { filterStateToQueryText } from "@/src/features/search-bar/lib/filter-state-to-query";+import { addDatasetNameObservedOptions } from "./datasetNameFilter";++const datasets = [+ { id: "dataset-id-1", name: "Support conversations" },+ { id: "dataset-id-2", name: "Billing: escalations" },+];+const withDatasets = (registry: FieldRegistry) =>+ withFieldOptions(+ registry,+ "datasetName",+ datasets.map((dataset) => ({+ value: dataset.id,+ displayValue: dataset.name,+ })),+ );++describe("dataset name search filters", () => {+ it("uses dedicated AI-aware sample-filter registries", () => {+ expect(EVALUATOR_FIELD_REGISTRY).toMatchObject({+ id: "evaluatorSamples",+ aiFilterPrompt: true,+ aiContextFields: expect.arrayContaining([+ {+ observedOptionsKey: "datasetName",+ promptLabel: "datasetName",+ },+ ]),+ });+ expect(RULE_SAMPLE_FIELD_REGISTRY).toMatchObject({+ id: "ruleSamples",+ aiFilterPrompt: true,+ aiContextFields: expect.arrayContaining([+ {+ observedOptionsKey: "datasetName",+ promptLabel: "datasetName",+ },+ ]),+ });+ });++ it("preserves the search bar's pending observed-options state", () => {+ expect(addDatasetNameObservedOptions(undefined, datasets)).toBeUndefined();+ expect(addDatasetNameObservedOptions({}, datasets)).toMatchObject({+ datasetName: [+ { value: "Support conversations" },+ { value: "Billing: escalations" },+ ],+ });+ });++ it("projects dataset IDs to names and resolves names back to IDs", () => {+ const persisted = [+ {+ column: "experimentDatasetId",+ type: "stringOptions",+ operator: "any of",+ value: ["dataset-id-1", "dataset-id-2"],+ },+ ] satisfies FilterState;++ const query = filterStateToQueryText(+ persisted,+ {},+ withDatasets(EVALUATOR_FIELD_REGISTRY),+ );+ expect(query.text).toBe(+ 'datasetName:("Support conversations" OR "Billing: escalations")',+ );+ const result = planCommit(+ query.text,+ undefined,+ withDatasets(EVALUATOR_FIELD_REGISTRY),+ );+ expect(result).toMatchObject({ status: "committed", filters: persisted });+ });++ it("keeps filters for deleted datasets lossless", () => {+ const persisted = [+ {+ column: "experimentDatasetId",+ type: "stringOptions",+ operator: "none of",+ value: ["deleted-dataset-id"],+ },+ ] satisfies FilterState;++ const registry = withDatasets(EVALUATOR_FIELD_REGISTRY);+ const query = filterStateToQueryText(persisted, {}, registry);+ expect(query.text).toBe("-experimentDatasetId:deleted-dataset-id");+ expect(planCommit(query.text, undefined, registry)).toMatchObject({+ status: "committed",+ filters: persisted,+ });+ });++ it("round-trips dataset presence filters through the name column", () => {+ const persisted = [+ {+ column: "experimentDatasetId",+ type: "null",+ operator: "is not null",+ value: "",+ },+ ] satisfies FilterState;++ const registry = withDatasets(EVALUATOR_FIELD_REGISTRY);+ expect(filterStateToQueryText(persisted, {}, registry).text).toBe(+ "has:datasetName",+ );+ const result = planCommit("has:datasetName", undefined, registry);+ expect(result).toMatchObject({ status: "committed", filters: persisted });+ });++ it("accepts dataset names in evaluator and rule search queries", () => {+ for (const registry of [+ EVALUATOR_FIELD_REGISTRY,+ RULE_SAMPLE_FIELD_REGISTRY,+ ]) {+ const registryWithDatasets = withDatasets(registry);+ const result = planCommit(+ 'datasetName:"Support conversations"',+ undefined,+ registryWithDatasets,+ );++ expect(result.status).toBe("committed");+ if (result.status !== "committed") continue;++ expect(result.filters).toEqual([+ {+ column: "experimentDatasetId",+ type: "stringOptions",+ operator: "any of",+ value: ["dataset-id-1"],+ },+ ]);+ }+ });++ it("rejects unresolved dataset names instead of persisting them as IDs", () => {+ const registry = withDatasets(EVALUATOR_FIELD_REGISTRY);+ const result = planCommit(+ 'datasetName:"Missing dataset"',+ undefined,+ registry,+ );++ expect(result.status).toBe("invalid");+ if (result.status === "invalid") {+ expect(result.diagnostics).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ message: '"Missing dataset" is not a valid dataset name',+ }),+ ]),+ );+ }+ });++ it("rejects partial-match operators for dataset names", () => {+ const registry = withFieldOptions(EVALUATOR_FIELD_REGISTRY, "datasetName", [+ { value: "dataset-id", displayValue: "Support" },+ ]);++ expect(+ planCommit("datasetName:*Support*", undefined, registry).status,+ ).toBe("invalid");+ });++ it("quotes dataset names containing grammar characters", () => {+ expect(+ filterStateToQueryText(+ [+ {+ column: "experimentDatasetId",+ type: "stringOptions",+ operator: "any of",+ value: ["dataset-id-2"],+ },+ ],+ {},+ withDatasets(EVALUATOR_FIELD_REGISTRY),+ ).text,+ ).toBe('datasetName:"Billing: escalations"');+ });++ it("keeps experimentDatasetName as a backward-compatible alias", () => {+ const result = planCommit(+ 'experimentDatasetName:"Support conversations"',+ undefined,+ withDatasets(EVALUATOR_FIELD_REGISTRY),+ );++ expect(result).toMatchObject({+ status: "committed",+ filters: [+ {+ column: "experimentDatasetId",+ value: ["dataset-id-1"],+ },+ ],+ });+ });+});diff --git a/web/src/features/evals/v2/utils/datasetNameFilter.ts b/web/src/features/evals/v2/utils/datasetNameFilter.tsnew file mode 100644index 0000000000..b79caf038d--- /dev/null+++ b/web/src/features/evals/v2/utils/datasetNameFilter.ts@@ -0,0 +1,38 @@+import type { ColumnDefinition } from "@langfuse/shared";++import type { ObservedOptions } from "@/src/features/search-bar/lib/observed-options";++export type DatasetFilterOption = {+ id: string;+ name: string;+};++export const DATASET_NAME_COLUMN = "datasetName";+export const DATASET_NAME_FILTER_COLUMN: ColumnDefinition = {+ name: "Dataset name",+ id: DATASET_NAME_COLUMN,+ aliases: [+ "dataset",+ "datasetname",+ "dataset_name",+ "experimentdatasetname",+ "experiment_dataset_name",+ ],+ type: "stringOptions",+ internal: "experiment_dataset_id",+ options: [],+ nullable: true,+};++export function addDatasetNameObservedOptions(+ observed: ObservedOptions | undefined,+ datasets: DatasetFilterOption[],+): ObservedOptions | undefined {+ if (observed === undefined) return undefined;+ return {+ ...observed,+ [DATASET_NAME_COLUMN]: datasets.map((dataset) => ({+ value: dataset.name,+ })),+ };+}diff --git a/web/src/features/search-bar/README.md b/web/src/features/search-bar/README.mdindex 423a16dcdc..4807e78eb2 100644--- a/web/src/features/search-bar/README.md+++ b/web/src/features/search-bar/README.md@@ -444,6 +444,11 @@ so a new view must check the field dropdown, not just that Enter applies.3. **Reuse the view's `filterOptions` tRPC** for observed values —`observed-options.ts` already maps that payload to per-column observedvalues; point it at the new view's procedure (do not invent a parallel one).+ When a field displays labels but persists stable values, declare its+ canonical `filterColumn` in the registry overlay and hydrate it with+ `withFieldOptions([{ value, displayValue }])`. The shared adapter then emits+ the canonical column/value while the reverse adapter renders the label.+ Do not add a host-specific post-lowering conversion.4. **Keep the adapter targeting the shared `FilterState`.** Reuse thealready-registry-driven `operatorIssue`/`negationIssue` and the existingper-kind lowering. Never add a second lowering path — that breaks thediff --git a/web/src/features/search-bar/components/SearchComposer.tsx b/web/src/features/search-bar/components/SearchComposer.tsxindex 1f699ba3bc..6457ce2046 100644--- a/web/src/features/search-bar/components/SearchComposer.tsx+++ b/web/src/features/search-bar/components/SearchComposer.tsx@@ -1515,7 +1515,7 @@ export function SearchComposer({</div>{showGlobalDiagnostics && (- <div className="absolute top-1.5 right-2 flex items-center gap-1">+ <div className="absolute top-1/2 right-2 flex -translate-y-1/2 items-center gap-1"><spanclassName="text-destructive"title={visibleDiagnostics.map((d) => d.message).join("; ")}diff --git a/web/src/features/search-bar/hooks/useEventsSearchBar.ts b/web/src/features/search-bar/hooks/useEventsSearchBar.tsindex a3a6d46184..7e6489694e 100644--- a/web/src/features/search-bar/hooks/useEventsSearchBar.ts+++ b/web/src/features/search-bar/hooks/useEventsSearchBar.ts@@ -111,11 +111,13 @@ export function useEventsSearchBar({// validation so both route `scores.<name>` by the same observed score type.const observedRef = useRef(observed);observedRef.current = observed;+ const registryRef = useRef(registry);+ registryRef.current = registry;const [store] = useState(() =>createSearchBarStore(() => scoreTypeContextFromObserved(observedRef.current),- registry,+ () => registryRef.current,),);@@ -144,17 +146,14 @@ export function useEventsSearchBar({store.getState().actions.resetTo(committedText);}, [enabled, committedText, store]);- // Re-validate when observed options load: a draft typed before score types- // were known has a stale draftValid (the editor's red-border gate reads it),- // so without this a `scores.<numeric>:<non-number>` typed during the load- // window would commit-reject with no visible error. `observed` identity is- // NOT stable across refetches (a relative range + auto-refresh rebuilds it- // every tick), so this can fire on ticks where the score types are unchanged- // — revalidate() bails on a set-equal context, keeping that path a no-op.+ // Re-validate when observed options or the registry load: a draft typed+ // before score types or dynamic allowed values were known has stale+ // draftValid. Their identities can rotate across refetches, so revalidate()+ // bails when both effective contexts are unchanged.useEffect(() => {if (!enabled) return;store.getState().actions.revalidate();- }, [enabled, observed, store]);+ }, [enabled, observed, registry, store]);// Latest applied-state setters, read inside commit without rebuilding it.const applyRef = useRef({ setFilterState, setSearchQuery, setSearchType });diff --git a/web/src/features/search-bar/lib/adapter.ts b/web/src/features/search-bar/lib/adapter.tsindex 920beed337..d5b6170ea1 100644--- a/web/src/features/search-bar/lib/adapter.ts+++ b/web/src/features/search-bar/lib/adapter.ts@@ -468,11 +468,18 @@ function lowerText(return;}if (field.syncMode === "exactOption") {+ const values = field.filterValueByDisplayValue+ ? node.values.map((value) => field.filterValueByDisplayValue!.get(value))+ : node.values;+ if (values.some((value) => value === undefined)) {+ errors.push(`"${field.id}" contains an unknown option`);+ return;+ }out.push({type: "stringOptions",- column: field.id,+ column: field.filterColumn ?? field.id,operator: negated ? "none of" : "any of",- value: node.values,+ value: values as string[],});return;}@@ -866,7 +873,7 @@ function lowerHas(}out.push({type: "null",- column: target.field.id,+ column: target.field.filterColumn ?? target.field.id,operator: negated ? "is null" : "is not null",value: "",});diff --git a/web/src/features/search-bar/lib/fields.ts b/web/src/features/search-bar/lib/fields.tsindex 80b5226f80..9aebb02719 100644--- a/web/src/features/search-bar/lib/fields.ts+++ b/web/src/features/search-bar/lib/fields.ts@@ -16,7 +16,11 @@// AUTOCOMPLETE picker independently — `id`/`name` are textSearch (so `id:abc`// is a substring search) yet keep their observed-value picker.-import { eventsTableCols, type ColumnDefinition } from "@langfuse/shared";+import {+ eventsTableCols,+ type ColumnDefinition,+ type SingleValueOption,+} from "@langfuse/shared";import type { CompareOp } from "./ast";import { quoteIfNeeded, unquote } from "./quoting";@@ -25,7 +29,8 @@ type FieldKind = "text" | "number" | "datetime" | "boolean";type SyncMode = "exactOption" | "arrayOption" | "textSearch";export type FieldDef = {- /** Canonical field id; also the filter `column` sent to the API. */+ /** Canonical query field id; defaults to the FilterState column unless+ * `filterColumn` maps a display-oriented field to canonical storage. */id: string;/** Lowercase aliases accepted by the grammar (canonical id always works too). */aliases: string[];@@ -52,12 +57,23 @@ export type FieldDef = {* `exactOption`; only set this on `textSearch` fields that should suggest.*/suggestObservedValues?: boolean;+ /** Canonical FilterState column emitted for this display-oriented field. */+ filterColumn?: string;+ /** Display/query value → canonical FilterState value for labeled options. */+ filterValueByDisplayValue?: ReadonlyMap<string, string>;+ /** Canonical FilterState value → display/query value for labeled options. */+ displayValueByFilterValue?: ReadonlyMap<string, string>;/** Nullable-only columns participate in `has:` but are not direct filters. */directFilter?: boolean;};export type FieldRegistry = {- id: "events" | "evaluationRules" | "sessions";+ id:+ | "events"+ | "evaluationRules"+ | "evaluatorSamples"+ | "ruleSamples"+ | "sessions";fields: readonly FieldDef[];columns: readonly ColumnDefinition[];allowFreeText: boolean;@@ -163,6 +179,9 @@ export function fieldRegistryFromColumns(nullable: column.nullable,directFilter: column.type !== "null",suggestObservedValues: fieldOverlay?.suggestObservedValues,+ filterColumn: fieldOverlay?.filterColumn,+ filterValueByDisplayValue: fieldOverlay?.filterValueByDisplayValue,+ displayValueByFilterValue: fieldOverlay?.displayValueByFilterValue,unit: fieldOverlay?.unit,negatedLabel: fieldOverlay?.negatedLabel,};@@ -184,6 +203,73 @@ export function fieldRegistryFromColumns(});}+export function extendFieldRegistryWithColumns(+ registry: FieldRegistry,+ columns: readonly ColumnDefinition[],+ fieldOverlays?: Readonly<Record<string, FieldOverlay>>,+): FieldRegistry {+ const addedFields = fieldRegistryFromColumns(columns, {+ id: registry.id,+ fields: fieldOverlays,+ }).fields;++ return createFieldRegistry({+ id: registry.id,+ fields: [...registry.fields, ...addedFields],+ columns: [...registry.columns, ...columns],+ metadata: registry.metadata,+ scores: registry.scores,+ traceScores: registry.traceScores,+ allowFreeText: registry.allowFreeText,+ defaultTextField: registry.defaultTextField,+ searchExamples: registry.searchExamples,+ recentSearches: registry.recentSearches,+ aiFilterPrompt: registry.aiFilterPrompt,+ aiContextFields: registry.aiContextFields,+ });+}++export function withFieldOptions(+ registry: FieldRegistry,+ fieldId: string,+ options: readonly SingleValueOption[],+): FieldRegistry {+ const filterValueByDisplayValue = new Map(+ options.map((option) => [+ option.displayValue ?? option.value,+ option.value,+ ]),+ );+ const displayValueByFilterValue = new Map(+ options.map((option) => [+ option.value,+ option.displayValue ?? option.value,+ ]),+ );+ return createFieldRegistry({+ id: registry.id,+ fields: registry.fields.map((field) =>+ field.id === fieldId+ ? {+ ...field,+ filterValueByDisplayValue,+ displayValueByFilterValue,+ }+ : field,+ ),+ columns: registry.columns,+ metadata: registry.metadata,+ scores: registry.scores,+ traceScores: registry.traceScores,+ allowFreeText: registry.allowFreeText,+ defaultTextField: registry.defaultTextField,+ searchExamples: registry.searchExamples,+ recentSearches: registry.recentSearches,+ aiFilterPrompt: registry.aiFilterPrompt,+ aiContextFields: registry.aiContextFields,+ });+}+// prettier-ignoreexport const FIELDS: FieldDef[] = [{ id: "id", aliases: ["spanid", "span_id", "observationid", "observation_id"], kind: "text", syncMode: "textSearch", suggestObservedValues: true, label: "Observation ID", description: "Observation/span identifier" },@@ -318,6 +404,11 @@ function createFieldRegistry({columnIds.set(alias.toLowerCase(), column.id);}}+ for (const field of fields) {+ if (field.filterColumn) {+ columnIds.set(field.filterColumn.toLowerCase(), field.id);+ }+ }const registry: FieldRegistry = {id,@@ -529,6 +620,9 @@ export function operatorIssue(if (f.directFilter === false) {return `"${f.id}" only supports presence checks (has:${f.id} or -has:${f.id})`;}+ if (f.filterColumn && (op === "~" || op === "^" || op === "$")) {+ return `"${f.id}" maps labeled options to exact stored values and does not support ${label(op)}`;+ }if (f.kind === "number") {if (op === "~" || op === "^" || op === "$") {return `"${f.id}" is a number field and does not support ${label(op)}`;diff --git a/web/src/features/search-bar/lib/filter-state-to-query.ts b/web/src/features/search-bar/lib/filter-state-to-query.tsindex e95cd4410c..f3fd91a22e 100644--- a/web/src/features/search-bar/lib/filter-state-to-query.ts+++ b/web/src/features/search-bar/lib/filter-state-to-query.ts@@ -73,15 +73,31 @@ function lowerSingle(switch (filter.type) {case "stringOptions":case "arrayOptions": {- const id = registry.columnIdOf(filter.column);+ let id = registry.columnIdOf(filter.column);if (id === null || filter.value.length === 0) return null;+ let values = filter.value;+ let ref = registry.resolveField(id);+ const displayValueByFilterValue =+ ref?.type === "field" ? ref.field.displayValueByFilterValue : undefined;+ if (displayValueByFilterValue !== undefined) {+ const displayValues = filter.value.map((value) =>+ displayValueByFilterValue.get(value),+ );+ if (displayValues.every((value) => value !== undefined)) {+ values = displayValues as string[];+ } else {+ // A deleted option has no label mapping. Keep its canonical field and+ // value so the filter remains visible and round-trips losslessly.+ id = filter.column;+ ref = registry.resolveField(id);+ }+ }// A stringOptions/arrayOptions filter is EXACT-set semantics. On a// `textSearch` field (id/name) the bar reads a bare single value as// `contains`, so a single-value any-of/none-of would silently flip// exact→substring on the next commit. The single-value forms therefore use// the explicit exact operator (`name:=abc` / `-name:=abc`); the grouped// multi-value forms already reparse to exact any-of/none-of.- const ref = registry.resolveField(id);const isTextSearch =ref?.type === "field" && ref.field.syncMode === "textSearch";if (filter.operator === "none of") {@@ -90,9 +106,9 @@ function lowerSingle(// none-of — NOT the bare `-name:abc`, which is does-not-contain// (substring). The grouped/option forms use the bare `=` any-of shape.if (isTextSearch && filter.value.length === 1) {- return negate(filterNode(id, "exact", filter.value));+ return negate(filterNode(id, "exact", values));}- return negate(filterNode(id, "=", filter.value));+ return negate(filterNode(id, "=", values));}if (filter.operator === "all of") {// A single-value all-of has no distinct grammar form — `(a)` reparses@@ -100,18 +116,22 @@ function lowerSingle(// the next commit. Skip it (preserved via skippedFilters) rather than// rewrite; multi-value all-of serializes to the `(a AND b)` group.if (filter.value.length < 2) return null;- return filterNode(id, "=", filter.value, "and");+ return filterNode(id, "=", values, "and");}// Single-value any-of on a textSearch field: emit the explicit exact form// (`id:=abc`) so it round-trips to `{string,=}` (exact preserved), not the// bare `id:abc` that would re-lower to `contains`.if (isTextSearch && filter.value.length === 1) {- return filterNode(id, "exact", filter.value);+ return filterNode(id, "exact", values);}- return filterNode(id, "=", filter.value);+ return filterNode(id, "=", values);}case "string": {- const id = registry.columnIdOf(filter.column);+ const directRef = registry.resolveField(filter.column);+ const id =+ directRef?.type === "field"+ ? directRef.field.id+ : registry.columnIdOf(filter.column);if (id === null) return null;if (filter.operator === "does not contain") {// Mirror the positive contains carve-out below: a textSearch field emitsdiff --git a/web/src/features/search-bar/lib/searchBarInvariants.clienttest.ts b/web/src/features/search-bar/lib/searchBarInvariants.clienttest.tsindex 118d93bb6e..be96432ba7 100644--- a/web/src/features/search-bar/lib/searchBarInvariants.clienttest.ts+++ b/web/src/features/search-bar/lib/searchBarInvariants.clienttest.ts@@ -2,8 +2,16 @@import { describe, expect, it } from "vitest";-import { EVENTS_FIELD_REGISTRY } from "./fields";+import {+ EVENTS_FIELD_REGISTRY,+ type FieldRegistry,+ withFieldOptions,+} from "./fields";import { RULE_FIELD_REGISTRY } from "@/src/features/evals/v2/constants/ruleSearchRegistry";+import {+ EVALUATOR_FIELD_REGISTRY,+ RULE_SAMPLE_FIELD_REGISTRY,+} from "@/src/features/evals/v2/constants/evaluatorSearchRegistry";import { SESSIONS_FIELD_REGISTRY } from "@/src/features/filters/config/sessionsSearchRegistry";import { validateQuery } from "./validate";import { planCommit } from "./commit";@@ -99,6 +107,59 @@ const evaluationRulesView: RegistryUnderTest = {freeTextValues: [],};+const withInvariantDatasetOptions = (+ registry: FieldRegistry,+ values: string[],+) =>+ withFieldOptions(+ registry,+ "datasetName",+ values.map((displayValue, index) => ({+ value: `dataset-${index}`,+ displayValue,+ })),+ );++const sampleFilterViews: RegistryUnderTest[] = [+ {+ ...eventsView,+ name: "evaluator sample filters",+ registry: withInvariantDatasetOptions(+ EVALUATOR_FIELD_REGISTRY,+ eventsView.fieldValues,+ ),+ },+ {+ ...evaluationRulesView,+ name: "rule sample filters",+ registry: withInvariantDatasetOptions(+ RULE_SAMPLE_FIELD_REGISTRY,+ evaluationRulesView.fieldValues,+ ),+ },+];++describe.each(sampleFilterViews)(+ "search bar invariants — $name registry",+ (view) => {+ it("holds parity, round-trip, and serialization invariants", () => {+ const failures = runSearchBarInvariants(view);+ expect(+ failures,+ failures.length === 0+ ? "ok"+ : `\n${failures+ .slice(0, 25)+ .map(+ (failure) =>+ ` [${failure.invariant}] ${failure.case} — ${failure.detail}`,+ )+ .join("\n")}`,+ ).toEqual([]);+ });+ },+);+describe("search bar invariants — events v4 registry", () => {it("generates a broad field × operator × value matrix", () => {// Sanity: the matrix actually exercises the registry (guards against adiff --git a/web/src/features/search-bar/lib/validate.ts b/web/src/features/search-bar/lib/validate.tsindex 96b614153b..0a4eb0e25b 100644--- a/web/src/features/search-bar/lib/validate.ts+++ b/web/src/features/search-bar/lib/validate.ts@@ -159,6 +159,43 @@ function incompleteFieldTokenDiagnostics(}}+function labeledOptionValueDiagnostics(+ node: ASTNode,+ textLength: number,+ out: Diagnostic[],+ registry: FieldRegistry,+): void {+ if (node.kind === "not") {+ labeledOptionValueDiagnostics(node.child, textLength, out, registry);+ return;+ }+ if (node.kind === "and" || node.kind === "or") {+ for (const child of node.children) {+ labeledOptionValueDiagnostics(child, textLength, out, registry);+ }+ return;+ }+ if (node.kind !== "filter") return;++ const ref = registry.resolveField(node.key);+ if (+ ref?.type !== "field" ||+ ref.field.filterValueByDisplayValue === undefined+ )+ return;++ for (const value of node.values) {+ if (ref.field.filterValueByDisplayValue.has(value)) continue;+ const span = nodeSpan(node, textLength);+ out.push({+ from: span.from,+ to: span.to,+ severity: "error",+ message: `"${value}" is not a valid ${ref.field.label.toLowerCase()}`,+ });+ }+}+export function semanticDiagnostics(ast: ASTNode | null,textLength: number,@@ -172,6 +209,7 @@ export function semanticDiagnostics(// filter, not free text — checked over the WHOLE tree (not per top-level// node) so the adjacency scoping can see each text node's siblings.incompleteFieldTokenDiagnostics(ast, out, registry);+ labeledOptionValueDiagnostics(ast, textLength, out, registry);// Lower each top-level node independently so error spans point at the// offending node instead of the whole query. The lowering must see the samediff --git a/web/src/features/search-bar/server/buildFilterPrompt.ts b/web/src/features/search-bar/server/buildFilterPrompt.tsindex 1a3f24b39c..4e8c87f291 100644--- a/web/src/features/search-bar/server/buildFilterPrompt.ts+++ b/web/src/features/search-bar/server/buildFilterPrompt.ts@@ -154,7 +154,7 @@ export function buildFilterSystemPrompt(const catalog = buildFieldCatalog(registry);const nullableIds = nullableFieldIds(registry);- if (registry.id === "evaluationRules") {+ if (registry.id === "evaluationRules" || registry.id === "ruleSamples") {return `You generate observation filters for a Langfuse evaluation rule.Respond with ONLY a JSON array of flat FilterState objects. Use only canonicalcolumns from this catalog and only the documented type/operator/value shape:diff --git a/web/src/features/search-bar/server/parseFilterCompletion.ts b/web/src/features/search-bar/server/parseFilterCompletion.tsindex de5281c8b0..98a387fba4 100644--- a/web/src/features/search-bar/server/parseFilterCompletion.ts+++ b/web/src/features/search-bar/server/parseFilterCompletion.ts@@ -11,6 +11,7 @@ import {SCORE_COLUMNS,type FieldRegistry,} from "../lib/fields";+import { planCommit } from "../lib/commit";import { filterStateToQueryText } from "../lib/filter-state-to-query";import type { ObservedScoreNames } from "../lib/observed-options";@@ -287,15 +288,32 @@ export function parseGeneratedFilters(compatible,scoreNames,);+ const projected: FilterState = [];+ for (const filter of scoreChecked) {+ const id = registry.columnIdOf(filter.column);+ const ref = id === null ? null : registry.resolveField(id);+ if (ref?.type !== "field" || ref.field.filterColumn === undefined) {+ projected.push(filter);+ continue;+ }++ const rendered = filterStateToQueryText([filter], {}, registry);+ const result = planCommit(rendered.text, undefined, registry);+ if (rendered.skippedFilters.length === 0 && result.status === "committed") {+ for (const canonicalFilter of result.filters) {+ projected.push(canonicalFilter);+ }+ }+ }// Guardrail 3: drop anything that doesn't round-trip to bar grammar (unknown /// non-representable columns land in skippedFilters).const { text, skippedFilters } = filterStateToQueryText(- scoreChecked,+ projected,{},registry,);const skipped = new Set(skippedFilters);- const filters = scoreChecked.filter((f) => !skipped.has(f));+ const filters = projected.filter((f) => !skipped.has(f));return {filters,queryText: text,diff --git a/web/src/features/search-bar/server/router.ts b/web/src/features/search-bar/server/router.tsindex 792d010fd3..1702075a9e 100644--- a/web/src/features/search-bar/server/router.ts+++ b/web/src/features/search-bar/server/router.ts@@ -49,8 +49,13 @@ import {} from "./parseOutcomeScoring";import { getProductBaseUrl } from "@/src/utils/base-url";import { RULE_FIELD_REGISTRY } from "@/src/features/evals/v2/constants/ruleSearchRegistry";+import {+ EVALUATOR_FIELD_REGISTRY,+ RULE_SAMPLE_FIELD_REGISTRY,+} from "@/src/features/evals/v2/constants/evaluatorSearchRegistry";+import { DATASET_NAME_COLUMN } from "@/src/features/evals/v2/utils/datasetNameFilter";import { SESSIONS_FIELD_REGISTRY } from "@/src/features/filters/config/sessionsSearchRegistry";-import { EVENTS_FIELD_REGISTRY } from "../lib/fields";+import { EVENTS_FIELD_REGISTRY, withFieldOptions } from "../lib/fields";// Caps shared with `observedScoreNamesFromOptions` (the client-side builder),// which sends a set as undefined instead of ever exceeding them.@@ -62,7 +67,13 @@ const GenerateFilterInput = z.object({projectId: z.string(),prompt: z.string().min(1).max(2048),registryId: z- .enum(["events", "evaluationRules", "sessions"])+ .enum([+ "events",+ "evaluationRules",+ "evaluatorSamples",+ "ruleSamples",+ "sessions",+ ]).default("events"),/** Existing bar query text, so the model refines the current filters. */currentQuery: z.string().max(4096).optional(),@@ -89,16 +100,20 @@ export const searchBarRouter = createTRPCRouter({.input(GenerateFilterInput).mutation(async ({ input, ctx }) => {try {- const registry =+ const baseRegistry =input.registryId === "evaluationRules"? RULE_FIELD_REGISTRY- : input.registryId === "sessions"- ? SESSIONS_FIELD_REGISTRY- : EVENTS_FIELD_REGISTRY;+ : input.registryId === "evaluatorSamples"+ ? EVALUATOR_FIELD_REGISTRY+ : input.registryId === "ruleSamples"+ ? RULE_SAMPLE_FIELD_REGISTRY+ : input.registryId === "sessions"+ ? SESSIONS_FIELD_REGISTRY+ : EVENTS_FIELD_REGISTRY;// Defence in depth for the client gate above: generating against the// fallback (events) prompt for a view that has no branch of its own// produces filters for columns that view does not have.- if (!registry.aiFilterPrompt) {+ if (!baseRegistry.aiFilterPrompt) {throw new TRPCError({code: "BAD_REQUEST",message: "AI filter generation is not available for this view",@@ -113,14 +128,25 @@ export const searchBarRouter = createTRPCRouter({scope: "project:read",});- const project = await ctx.prisma.project.findUnique({- where: { id: input.projectId },- select: {- organization: {- select: { aiFeaturesEnabled: true, aiTelemetryEnabled: true },+ const isSampleRegistry =+ input.registryId === "evaluatorSamples" ||+ input.registryId === "ruleSamples";+ const [project, datasets] = await Promise.all([+ ctx.prisma.project.findUnique({+ where: { id: input.projectId },+ select: {+ organization: {+ select: { aiFeaturesEnabled: true, aiTelemetryEnabled: true },+ },},- },- });+ }),+ isSampleRegistry+ ? ctx.prisma.dataset.findMany({+ where: { projectId: input.projectId },+ select: { id: true, name: true },+ })+ : Promise.resolve([]),+ ]);if (!project) {throw new TRPCError({@@ -136,6 +162,17 @@ export const searchBarRouter = createTRPCRouter({});}+ const registry = isSampleRegistry+ ? withFieldOptions(+ baseRegistry,+ DATASET_NAME_COLUMN,+ datasets.map((dataset) => ({+ value: dataset.id,+ displayValue: dataset.name,+ })),+ )+ : baseRegistry;+const modelConfig = getInAppAgentModelConfig();if (!modelConfig) {diff --git a/web/src/features/search-bar/store/searchBarStore.clienttest.ts b/web/src/features/search-bar/store/searchBarStore.clienttest.tsindex f4d76c564a..4a9cda8fba 100644--- a/web/src/features/search-bar/store/searchBarStore.clienttest.ts+++ b/web/src/features/search-bar/store/searchBarStore.clienttest.ts@@ -1,4 +1,6 @@import { createSearchBarStore } from "@/src/features/search-bar/store/searchBarStore";+import { EVALUATOR_FIELD_REGISTRY } from "@/src/features/evals/v2/constants/evaluatorSearchRegistry";+import { withFieldOptions } from "@/src/features/search-bar/lib/fields";describe("searchBarStore (draft-only)", () => {it("validates the draft on setDraft", () => {@@ -242,6 +244,24 @@ describe("searchBarStore (draft-only)", () => {expect(store.getState().draftValid).toBe(false);});+ it("revalidates drafts against the latest dynamic registry", () => {+ let registry = withFieldOptions(+ EVALUATOR_FIELD_REGISTRY,+ "datasetName",+ [],+ );+ const store = createSearchBarStore(undefined, () => registry);+ store.getState().actions.setDraft('datasetName:"Filter QA Dataset"');+ expect(store.getState().draftValid).toBe(false);++ registry = withFieldOptions(EVALUATOR_FIELD_REGISTRY, "datasetName", [+ { value: "dataset-id", displayValue: "Filter QA Dataset" },+ ]);+ store.getState().actions.revalidate();++ expect(store.getState().draftValid).toBe(true);+ });+it("revalidate reveals the red state when a valid draft flips invalid", () => {// Committed against an empty context while filterOptions loaded; once score// types arrive and unmask it, the bar must paint red (not stay silentlydiff --git a/web/src/features/search-bar/store/searchBarStore.ts b/web/src/features/search-bar/store/searchBarStore.tsindex 9061632ea0..dbd5ffaf1f 100644--- a/web/src/features/search-bar/store/searchBarStore.ts+++ b/web/src/features/search-bar/store/searchBarStore.ts@@ -99,26 +99,35 @@ export type SearchBarStoreState = {export type SearchBarStore = StoreApi<SearchBarStoreState>;/**- * `resolveScoreTypes` lets draft validation route `scores.<name>` by observed- * score type — the SAME context planCommit uses — so the store's draftValid- * (which the editor's red-border gate reads) never disagrees with the commit- * gate. A thunk so it always reads the latest observed options.+ * The resolver thunks keep draft validation aligned with the same live score+ * context and field registry that planCommit uses. This matters for registries+ * whose allowed values arrive asynchronously.*/export function createSearchBarStore(resolveScoreTypes?: () => ScoreTypeContext | undefined,- registry: FieldRegistry = EVENTS_FIELD_REGISTRY,+ registryOrResolver:+ | FieldRegistry+ | (() => FieldRegistry) = EVENTS_FIELD_REGISTRY,): SearchBarStore {+ const resolveRegistry =+ typeof registryOrResolver === "function"+ ? registryOrResolver+ : () => registryOrResolver;+return createStore<SearchBarStoreState>((set, get) => {// The score-type context used by the most recent validation. revalidate()// bails when the context is set-equal to this, so observed-identity churn// (a relative range + auto-refresh rebuilds the context every tick) doesn't// re-parse the draft and emit a fresh diagnostics array for no change.let lastScoreTypes: ScoreTypeContext | undefined;+ let lastRegistry: FieldRegistry | undefined;let hasValidated = false;const writeDraft = (next: string) => {const scoreTypes = resolveScoreTypes?.();+ const registry = resolveRegistry();lastScoreTypes = scoreTypes;+ lastRegistry = registry;hasValidated = true;const res = validateQuery(next, scoreTypes, registry);set({@@ -159,6 +168,7 @@ export function createSearchBarStore(// Number-normalize numeric (not categorical) score values. It preserves// structure/order, so free-text canonicalization is kept.const scoreTypes = resolveScoreTypes?.();+ const registry = resolveRegistry();if (draftsSemanticallyEqual(committedText, draft, scoreTypes, registry))@@ -166,6 +176,7 @@ export function createSearchBarStore(writeDraft(committedText);},removeChipSpan: (from, to) => {+ const registry = resolveRegistry();const next = removeToken(get().draft, { from, to }, registry);writeDraft(next);return next;@@ -181,13 +192,19 @@ export function createSearchBarStore(},revalidate: () => {const scoreTypes = resolveScoreTypes?.();+ const registry = resolveRegistry();// Nothing to refresh: the draft was already validated against an- // equal context (writeDraft and revalidate both record it), so the- // result would be identical. Skip the parse + set to avoid a- // no-op re-render from the fresh diagnostics array.- if (hasValidated && scoreTypeContextEqual(scoreTypes, lastScoreTypes))+ // equal context and registry (writeDraft and revalidate both record+ // them), so the result would be identical. Skip the parse + set to+ // avoid a no-op re-render from the fresh diagnostics array.+ if (+ hasValidated &&+ registry === lastRegistry &&+ scoreTypeContextEqual(scoreTypes, lastScoreTypes)+ )return;lastScoreTypes = scoreTypes;+ lastRegistry = registry;hasValidated = true;const wasValid = get().draftValid;const res = validateQuery(get().draft, scoreTypes, registry);