Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(evals): filter evaluator samples by dataset name (#16799)

Tobias Wochingerb5a9ed2ebf63
diff --git a/web/src/__tests__/server/ai-filter-access.servertest.ts b/web/src/__tests__/server/ai-filter-access.servertest.ts
index ec2b4d128e..d89b2c2f1f 100644
--- a/web/src/__tests__/server/ai-filter-access.servertest.ts
+++ b/web/src/__tests__/server/ai-filter-access.servertest.ts
@@ -187,7 +187,44 @@ describe("Ask AI filter generation access", () => {
prompt: "traces from today",
}),
).resolves.toMatchObject({ filters: [] });
- expect(llmMocks.generateLLMText).toHaveBeenCalledOnce();
+
+ await prisma.dataset.create({
+ data: {
+ id: "ai-filter-dataset",
+ projectId: project.id,
+ name: "Filter QA Dataset",
+ },
+ });
+ llmMocks.generateLLMText.mockResolvedValue({
+ text: JSON.stringify([
+ {
+ type: "stringOptions",
+ column: "datasetName",
+ operator: "any of",
+ value: ["Filter QA Dataset"],
+ },
+ ]),
+ });
+
+ for (const registryId of ["evaluatorSamples", "ruleSamples"] as const) {
+ await expect(
+ caller.searchBar.generateFilter({
+ projectId: project.id,
+ prompt: "only the Filter QA Dataset",
+ registryId,
+ }),
+ ).resolves.toMatchObject({
+ filters: [
+ {
+ type: "stringOptions",
+ column: "experimentDatasetId",
+ operator: "any of",
+ value: ["ai-filter-dataset"],
+ },
+ ],
+ });
+ }
+ expect(llmMocks.generateLLMText).toHaveBeenCalledTimes(3);
} finally {
(
env as { NEXT_PUBLIC_LANGFUSE_CLOUD_REGION?: string }
diff --git a/web/src/__tests__/server/unit/searchBarFilterPrompt.servertest.ts b/web/src/__tests__/server/unit/searchBarFilterPrompt.servertest.ts
index ef064279b2..d6328e2aaf 100644
--- a/web/src/__tests__/server/unit/searchBarFilterPrompt.servertest.ts
+++ b/web/src/__tests__/server/unit/searchBarFilterPrompt.servertest.ts
@@ -5,6 +5,7 @@ import {
FIELDS,
SCORE_COLUMNS,
type FieldDef,
+ withFieldOptions,
} from "@/src/features/search-bar/lib/fields";
import {
buildFilterContextMessage,
@@ -13,6 +14,10 @@ import {
} from "@/src/features/search-bar/server/buildFilterPrompt";
import { filterStateToQueryText } from "@/src/features/search-bar/lib/filter-state-to-query";
import { RULE_FIELD_REGISTRY } from "@/src/features/evals/v2/constants/ruleSearchRegistry";
+import {
+ EVALUATOR_FIELD_REGISTRY,
+ RULE_SAMPLE_FIELD_REGISTRY,
+} from "@/src/features/evals/v2/constants/evaluatorSearchRegistry";
import { parseGeneratedFilters } from "@/src/features/search-bar/server/parseFilterCompletion";
// A representative `singleFilter` for the FilterState `type` the prompt tells
@@ -139,6 +144,61 @@ describe("evaluation rule AI registry", () => {
});
});
+describe("sample-filter AI registries", () => {
+ const options = [{ value: "dataset-id", displayValue: "Filter QA Dataset" }];
+
+ it("uses rule-specific prompt semantics for rule samples", () => {
+ const prompt = buildFilterSystemPrompt(
+ "Monday, 2026-06-15T00:00:00.000Z",
+ RULE_SAMPLE_FIELD_REGISTRY,
+ );
+
+ expect(prompt).toContain(
+ "observation filters for a Langfuse evaluation rule",
+ );
+ expect(prompt).toContain("- datasetName");
+ expect(prompt).not.toContain("observability (v4 events) table");
+ });
+
+ it("projects known names to IDs and drops unknown AI values", () => {
+ const registry = withFieldOptions(
+ EVALUATOR_FIELD_REGISTRY,
+ "datasetName",
+ options,
+ );
+ const completion = (value: string) =>
+ JSON.stringify([
+ {
+ type: "stringOptions",
+ column: "datasetName",
+ operator: "any of",
+ value: [value],
+ },
+ ]);
+
+ expect(
+ parseGeneratedFilters(
+ completion("Filter QA Dataset"),
+ undefined,
+ registry,
+ ),
+ ).toMatchObject({
+ filters: [
+ {
+ type: "stringOptions",
+ column: "experimentDatasetId",
+ operator: "any of",
+ value: ["dataset-id"],
+ },
+ ],
+ droppedCount: 0,
+ });
+ expect(
+ parseGeneratedFilters(completion("Missing dataset"), undefined, registry),
+ ).toMatchObject({ filters: [], droppedCount: 1 });
+ });
+});
+
describe("buildFilterContextMessage", () => {
it("returns null when neither current query nor data context is given", () => {
expect(buildFilterContextMessage()).toBeNull();
diff --git a/web/src/features/evals/hooks/useEvalConfigFilterOptions.clienttest.ts b/web/src/features/evals/hooks/useEvalConfigFilterOptions.clienttest.ts
index e57961ad39..84e0dc9e9f 100644
--- a/web/src/features/evals/hooks/useEvalConfigFilterOptions.clienttest.ts
+++ b/web/src/features/evals/hooks/useEvalConfigFilterOptions.clienttest.ts
@@ -6,6 +6,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
import { useEvalConfigFilterOptions } from "./useEvalConfigFilterOptions";
const mocks = vi.hoisted(() => ({
+ datasetsData: [] as Array<{ id: string; name: string }>,
eventsData: {} as Record<string, unknown>,
eventsInput: undefined as unknown,
eventsOptions: undefined as unknown,
@@ -19,7 +20,7 @@ vi.mock("@/src/utils/api", () => ({
api: {
datasets: {
allDatasetMeta: {
- useQuery: () => ({ data: [] }),
+ useQuery: () => ({ data: mocks.datasetsData }),
},
},
events: {
@@ -57,6 +58,7 @@ vi.mock("@/src/utils/api", () => ({
describe("useEvalConfigFilterOptions", () => {
beforeEach(() => {
+ mocks.datasetsData = [];
mocks.eventsData = {};
mocks.eventsInput = undefined;
mocks.eventsOptions = undefined;
@@ -136,4 +138,25 @@ describe("useEvalConfigFilterOptions", () => {
calledToolNames: [{ value: "legacy-tool" }],
});
});
+
+ it("shows dataset names while keeping IDs in legacy evaluator filters", () => {
+ mocks.datasetsData = [{ id: "dataset-id", name: "Filter QA Dataset" }];
+
+ const { result } = renderHook(() =>
+ useEvalConfigFilterOptions({
+ projectId: "project-id",
+ useEventsTable: false,
+ includeLegacyTraceOptions: false,
+ }),
+ );
+
+ expect(result.current.experimentEvalFilterOptions).toEqual({
+ experimentDatasetId: [
+ {
+ value: "dataset-id",
+ displayValue: "Filter QA Dataset",
+ },
+ ],
+ });
+ });
});
diff --git a/web/src/features/evals/v2/components/Evaluators/Testing/components/EvaluatorSampleObservationSelector/EvaluatorSampleObservationSelector.tsx b/web/src/features/evals/v2/components/Evaluators/Testing/components/EvaluatorSampleObservationSelector/EvaluatorSampleObservationSelector.tsx
index 64b6514c91..9c06bb125e 100644
--- a/web/src/features/evals/v2/components/Evaluators/Testing/components/EvaluatorSampleObservationSelector/EvaluatorSampleObservationSelector.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/Testing/components/EvaluatorSampleObservationSelector/EvaluatorSampleObservationSelector.tsx
@@ -10,6 +10,7 @@ import {
SampleObservationSelectorBase,
type SampleObservation,
} from "@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/SampleObservationSelectorBase";
+import { EVALUATOR_FIELD_REGISTRY } from "@/src/features/evals/v2/constants/evaluatorSearchRegistry";
const preserveObservedOptions: Parameters<
typeof SampleObservationSelectorBase
@@ -85,7 +86,7 @@ export function EvaluatorSampleObservationSelector({
onFilterStateChange={onFilterStateChange}
previewFilters={filterState}
tableName="evaluator-sample-observations"
- registry={undefined}
+ registry={EVALUATOR_FIELD_REGISTRY}
selectedObservationId={selectedObservationId}
onSelect={onSelect}
onOpenTrace={onOpenTrace}
diff --git a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/SampleObservationSelectorBase.tsx b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/SampleObservationSelectorBase.tsx
index 6b5952f32f..2d4d1ddde1 100644
--- a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/SampleObservationSelectorBase.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/SampleObservationSelectorBase.tsx
@@ -13,6 +13,7 @@ import { DataTableColumnVisibilityFilter } from "@/src/components/table/data-tab
import { useRowHeightLocalStorage } from "@/src/components/table/data-table-row-height-switch";
import type { LangfuseColumnDef } from "@/src/components/table/types";
import { Button } from "@/src/components/ui/button";
+import { Skeleton } from "@/src/components/ui/skeleton";
import { useEventsFilterOptions } from "@/src/features/events/hooks/useEventsFilterOptions";
import { EventsSearchBarRow } from "@/src/features/search-bar/components/EventsSearchBarRow";
import { useEventsSearchBar } from "@/src/features/search-bar/hooks/useEventsSearchBar";
@@ -20,6 +21,7 @@ import { buildAiContext } from "@/src/features/search-bar/lib/ai-context";
import {
type FieldRegistry,
EVENTS_FIELD_REGISTRY,
+ withFieldOptions,
} from "@/src/features/search-bar/lib/fields";
import {
observedScoreNamesFromOptions,
@@ -33,6 +35,10 @@ import { SectionHeader } from "@/src/features/evals/v2/components/Evaluators/Tes
import { EXPERIMENTS_AND_EVALS_EXCLUSION_FILTERS } from "@/src/features/evals/v2/constants/experimentAndEvalFilters";
import { dedupeObservationPages } from "@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/dedupeObservations";
import { toggleExampleFilters } from "@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/toggleExampleFilters";
+import {
+ DATASET_NAME_COLUMN,
+ addDatasetNameObservedOptions,
+} from "@/src/features/evals/v2/utils/datasetNameFilter";
import { useReusableRuleFilterPresets } from "@/src/features/evals/v2/hooks/useReusableRuleFilterPresets";
import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
@@ -171,9 +177,31 @@ export function SampleObservationSelectorBase(
mapObservedOptions,
} = props;
const activeRegistry = registry ?? EVENTS_FIELD_REGISTRY;
+ const datasets = api.datasets.allDatasetMeta.useQuery(
+ { projectId },
+ {
+ refetchOnMount: false,
+ refetchOnWindowFocus: false,
+ refetchOnReconnect: false,
+ staleTime: Infinity,
+ },
+ );
+ const datasetOptions = useMemo(() => datasets.data ?? [], [datasets.data]);
+ const searchRegistry = useMemo(
+ () =>
+ withFieldOptions(
+ activeRegistry,
+ DATASET_NAME_COLUMN,
+ datasetOptions.map((dataset) => ({
+ value: dataset.id,
+ displayValue: dataset.name,
+ })),
+ ),
+ [activeRegistry, datasetOptions],
+ );
const reusableRuleFilters = useReusableRuleFilterPresets(
projectId,
- activeRegistry,
+ searchRegistry,
);
const capture = usePostHogClientCapture();
const onQueryPresetPick = useCallback(
@@ -222,20 +250,21 @@ export function SampleObservationSelectorBase(
includeApproxCount: true,
lazy: true,
});
- const observed = useMemo(
- () =>
- mapObservedOptions(
- toObservedOptions(
- options.filterOptions,
- options.isFilterOptionsPending,
- ),
- ),
- [mapObservedOptions, options.filterOptions, options.isFilterOptionsPending],
- );
+ const observed = useMemo(() => {
+ const mapped = mapObservedOptions(
+ toObservedOptions(options.filterOptions, options.isFilterOptionsPending),
+ );
+ return addDatasetNameObservedOptions(mapped, datasetOptions);
+ }, [
+ datasetOptions,
+ mapObservedOptions,
+ options.filterOptions,
+ options.isFilterOptionsPending,
+ ]);
const search = useEventsSearchBar({
projectId,
tableName,
- enabled: true,
+ enabled: !datasets.isPending,
filterState,
searchQuery,
searchType,
@@ -249,7 +278,7 @@ export function SampleObservationSelectorBase(
setSearchType: (next) => {
setSearchType(next);
},
- ...(registry ? { registry } : {}),
+ registry: searchRegistry,
});
// listCursor always reads in the events table's stable start_time DESC tuple
// order, so it takes no orderBy.
@@ -329,10 +358,10 @@ export function SampleObservationSelectorBase(
resultCount: observationQuery.isSuccess
? matchingObservations.length
: null,
- registry: activeRegistry,
+ registry: searchRegistry,
}),
[
- activeRegistry,
+ searchRegistry,
matchingObservations,
observationQuery.isSuccess,
observed,
@@ -340,10 +369,10 @@ export function SampleObservationSelectorBase(
);
const aiScoreNames = useMemo(
() =>
- activeRegistry.scores
+ searchRegistry.scores
? observedScoreNamesFromOptions(observed)
: undefined,
- [activeRegistry, observed],
+ [searchRegistry, observed],
);
const selectionToReconcile = observationQuery.isSuccess
? resolveSelection(matchingObservations, selectedObservationId)
@@ -465,22 +494,26 @@ export function SampleObservationSelectorBase(
tooltip={filterTooltip}
trailing={null}
/>
- <EventsSearchBarRow
- projectId={projectId}
- tableName={tableName}
- store={search.store}
- commit={search.commit}
- observed={observed}
- erroredColumns={options.erroredColumns}
- {...(registry ? { registry } : {})}
- onApplyFilters={search.applyFilters}
- onRequestColumns={options.requestColumns}
- presetSections={reusableRuleFilters.sections}
- onQueryPresetPick={onQueryPresetPick}
- aiDataContext={aiDataContext}
- aiScoreNames={aiScoreNames}
- className="p-0"
- />
+ {datasets.isPending ? (
+ <Skeleton className="h-10 w-full" />
+ ) : (
+ <EventsSearchBarRow
+ projectId={projectId}
+ tableName={tableName}
+ store={search.store}
+ commit={search.commit}
+ observed={observed}
+ erroredColumns={options.erroredColumns}
+ registry={searchRegistry}
+ onApplyFilters={search.applyFilters}
+ onRequestColumns={options.requestColumns}
+ presetSections={reusableRuleFilters.sections}
+ onQueryPresetPick={onQueryPresetPick}
+ aiDataContext={aiDataContext}
+ aiScoreNames={aiScoreNames}
+ className="p-0"
+ />
+ )}
<div className="flex flex-wrap gap-2">
{EXAMPLES.map((example) => (
<Button
diff --git a/web/src/features/evals/v2/components/Rules/RuleSetup/components/RuleFilterStep.tsx b/web/src/features/evals/v2/components/Rules/RuleSetup/components/RuleFilterStep.tsx
index 9de2ba67db..808412e136 100644
--- a/web/src/features/evals/v2/components/Rules/RuleSetup/components/RuleFilterStep.tsx
+++ b/web/src/features/evals/v2/components/Rules/RuleSetup/components/RuleFilterStep.tsx
@@ -4,7 +4,7 @@ import { useStore } from "zustand";
import { RuleSampleObservationSelector } from "@/src/features/evals/v2/components/Evaluators/Testing/components/RuleSampleObservationSelector/RuleSampleObservationSelector";
import { Stepper } from "@/src/features/evals/v2/components/Stepper/Stepper";
import type { RuleSetupStore } from "@/src/features/evals/v2/types/rules";
-import { RULE_FIELD_REGISTRY } from "@/src/features/evals/v2/constants/ruleSearchRegistry";
+import { RULE_SAMPLE_FIELD_REGISTRY } from "@/src/features/evals/v2/constants/evaluatorSearchRegistry";
import { env } from "@/src/env.mjs";
import { RuleSamplingSection } from "./RuleSamplingSection";
@@ -38,7 +38,7 @@ export function RuleFilterStep({
filterState={filter}
onFilterStateChange={actions.setFilter}
tableName="evaluation-rule-matching-observations"
- registry={RULE_FIELD_REGISTRY}
+ registry={RULE_SAMPLE_FIELD_REGISTRY}
selectedObservationId={selectedObservationId}
onSelect={actions.setSelectedObservation}
onOpenTrace={(observation) => {
diff --git a/web/src/features/evals/v2/constants/evaluatorSearchRegistry.ts b/web/src/features/evals/v2/constants/evaluatorSearchRegistry.ts
new file mode 100644
index 0000000000..222623a0f9
--- /dev/null
+++ b/web/src/features/evals/v2/constants/evaluatorSearchRegistry.ts
@@ -0,0 +1,39 @@
+import { DATASET_NAME_FILTER_COLUMN } from "@/src/features/evals/v2/utils/datasetNameFilter";
+import { RULE_FIELD_REGISTRY } from "@/src/features/evals/v2/constants/ruleSearchRegistry";
+import {
+ EVENTS_FIELD_REGISTRY,
+ extendFieldRegistryWithColumns,
+ type FieldRegistry,
+} from "@/src/features/search-bar/lib/fields";
+
+const withDatasetNameAiContext = (
+ registry: FieldRegistry,
+): FieldRegistry["aiContextFields"] =>
+ registry.aiContextFields
+ .filter(
+ ({ observedOptionsKey }) => observedOptionsKey !== "experimentDatasetId",
+ )
+ .concat({
+ observedOptionsKey: "datasetName",
+ promptLabel: "datasetName",
+ });
+
+export const EVALUATOR_FIELD_REGISTRY = {
+ ...extendFieldRegistryWithColumns(
+ EVENTS_FIELD_REGISTRY,
+ [DATASET_NAME_FILTER_COLUMN],
+ { datasetName: { filterColumn: "experimentDatasetId" } },
+ ),
+ id: "evaluatorSamples" as const,
+ aiContextFields: withDatasetNameAiContext(EVENTS_FIELD_REGISTRY),
+};
+
+export const RULE_SAMPLE_FIELD_REGISTRY = {
+ ...extendFieldRegistryWithColumns(
+ RULE_FIELD_REGISTRY,
+ [DATASET_NAME_FILTER_COLUMN],
+ { datasetName: { filterColumn: "experimentDatasetId" } },
+ ),
+ id: "ruleSamples" as const,
+ aiContextFields: withDatasetNameAiContext(RULE_FIELD_REGISTRY),
+};
diff --git a/web/src/features/evals/v2/hooks/useReusableRuleFilterPresets.clienttest.ts b/web/src/features/evals/v2/hooks/useReusableRuleFilterPresets.clienttest.ts
index 35dd9c2072..fb550b7a77 100644
--- a/web/src/features/evals/v2/hooks/useReusableRuleFilterPresets.clienttest.ts
+++ b/web/src/features/evals/v2/hooks/useReusableRuleFilterPresets.clienttest.ts
@@ -4,8 +4,12 @@ import { describe, expect, it } from "vitest";
import type { FilterState } from "@langfuse/shared";
import { RULE_FIELD_REGISTRY } from "@/src/features/evals/v2/constants/ruleSearchRegistry";
+import { RULE_SAMPLE_FIELD_REGISTRY } from "@/src/features/evals/v2/constants/evaluatorSearchRegistry";
import { prepareReusableRuleFilterPresets } from "@/src/features/evals/v2/hooks/useReusableRuleFilterPresets";
-import { EVENTS_FIELD_REGISTRY } from "@/src/features/search-bar/lib/fields";
+import {
+ EVENTS_FIELD_REGISTRY,
+ withFieldOptions,
+} from "@/src/features/search-bar/lib/fields";
const reusableFilters = [
{
@@ -87,4 +91,32 @@ describe("prepareReusableRuleFilterPresets", () => {
expect(events.sections[0]?.options[0]?.query).toBe("name:=checkout");
expect(rules.sections[0]?.options[0]?.query).toBe("name:checkout");
});
+
+ it("renders dataset-scoped presets with names instead of IDs", () => {
+ const datasetFilters = [
+ {
+ latestRuleId: "rule-dataset",
+ filter: [
+ {
+ column: "experimentDatasetId",
+ type: "stringOptions",
+ operator: "any of",
+ value: ["dataset-id"],
+ },
+ ] satisfies FilterState,
+ evaluatorCount: 1,
+ updatedAt: new Date("2026-02-01T00:00:00.000Z"),
+ },
+ ];
+ const registry = withFieldOptions(
+ RULE_SAMPLE_FIELD_REGISTRY,
+ "datasetName",
+ [{ value: "dataset-id", displayValue: "Filter QA Dataset" }],
+ );
+
+ expect(
+ prepareReusableRuleFilterPresets(datasetFilters, registry).sections[0]
+ ?.options[0]?.query,
+ ).toBe('datasetName:"Filter QA Dataset"');
+ });
});
diff --git a/web/src/features/evals/v2/utils/datasetNameFilter.clienttest.ts b/web/src/features/evals/v2/utils/datasetNameFilter.clienttest.ts
new file mode 100644
index 0000000000..50519820d5
--- /dev/null
+++ b/web/src/features/evals/v2/utils/datasetNameFilter.clienttest.ts
@@ -0,0 +1,217 @@
+import type { FilterState } from "@langfuse/shared";
+import { describe, expect, it } from "vitest";
+
+import {
+ EVALUATOR_FIELD_REGISTRY,
+ RULE_SAMPLE_FIELD_REGISTRY,
+} from "@/src/features/evals/v2/constants/evaluatorSearchRegistry";
+import { planCommit } from "@/src/features/search-bar/lib/commit";
+import {
+ type FieldRegistry,
+ withFieldOptions,
+} from "@/src/features/search-bar/lib/fields";
+import { filterStateToQueryText } from "@/src/features/search-bar/lib/filter-state-to-query";
+import { addDatasetNameObservedOptions } from "./datasetNameFilter";
+
+const datasets = [
+ { id: "dataset-id-1", name: "Support conversations" },
+ { id: "dataset-id-2", name: "Billing: escalations" },
+];
+const withDatasets = (registry: FieldRegistry) =>
+ withFieldOptions(
+ registry,
+ "datasetName",
+ datasets.map((dataset) => ({
+ value: dataset.id,
+ displayValue: dataset.name,
+ })),
+ );
+
+describe("dataset name search filters", () => {
+ it("uses dedicated AI-aware sample-filter registries", () => {
+ expect(EVALUATOR_FIELD_REGISTRY).toMatchObject({
+ id: "evaluatorSamples",
+ aiFilterPrompt: true,
+ aiContextFields: expect.arrayContaining([
+ {
+ observedOptionsKey: "datasetName",
+ promptLabel: "datasetName",
+ },
+ ]),
+ });
+ expect(RULE_SAMPLE_FIELD_REGISTRY).toMatchObject({
+ id: "ruleSamples",
+ aiFilterPrompt: true,
+ aiContextFields: expect.arrayContaining([
+ {
+ observedOptionsKey: "datasetName",
+ promptLabel: "datasetName",
+ },
+ ]),
+ });
+ });
+
+ it("preserves the search bar's pending observed-options state", () => {
+ expect(addDatasetNameObservedOptions(undefined, datasets)).toBeUndefined();
+ expect(addDatasetNameObservedOptions({}, datasets)).toMatchObject({
+ datasetName: [
+ { value: "Support conversations" },
+ { value: "Billing: escalations" },
+ ],
+ });
+ });
+
+ it("projects dataset IDs to names and resolves names back to IDs", () => {
+ const persisted = [
+ {
+ column: "experimentDatasetId",
+ type: "stringOptions",
+ operator: "any of",
+ value: ["dataset-id-1", "dataset-id-2"],
+ },
+ ] satisfies FilterState;
+
+ const query = filterStateToQueryText(
+ persisted,
+ {},
+ withDatasets(EVALUATOR_FIELD_REGISTRY),
+ );
+ expect(query.text).toBe(
+ 'datasetName:("Support conversations" OR "Billing: escalations")',
+ );
+ const result = planCommit(
+ query.text,
+ undefined,
+ withDatasets(EVALUATOR_FIELD_REGISTRY),
+ );
+ expect(result).toMatchObject({ status: "committed", filters: persisted });
+ });
+
+ it("keeps filters for deleted datasets lossless", () => {
+ const persisted = [
+ {
+ column: "experimentDatasetId",
+ type: "stringOptions",
+ operator: "none of",
+ value: ["deleted-dataset-id"],
+ },
+ ] satisfies FilterState;
+
+ const registry = withDatasets(EVALUATOR_FIELD_REGISTRY);
+ const query = filterStateToQueryText(persisted, {}, registry);
+ expect(query.text).toBe("-experimentDatasetId:deleted-dataset-id");
+ expect(planCommit(query.text, undefined, registry)).toMatchObject({
+ status: "committed",
+ filters: persisted,
+ });
+ });
+
+ it("round-trips dataset presence filters through the name column", () => {
+ const persisted = [
+ {
+ column: "experimentDatasetId",
+ type: "null",
+ operator: "is not null",
+ value: "",
+ },
+ ] satisfies FilterState;
+
+ const registry = withDatasets(EVALUATOR_FIELD_REGISTRY);
+ expect(filterStateToQueryText(persisted, {}, registry).text).toBe(
+ "has:datasetName",
+ );
+ const result = planCommit("has:datasetName", undefined, registry);
+ expect(result).toMatchObject({ status: "committed", filters: persisted });
+ });
+
+ it("accepts dataset names in evaluator and rule search queries", () => {
+ for (const registry of [
+ EVALUATOR_FIELD_REGISTRY,
+ RULE_SAMPLE_FIELD_REGISTRY,
+ ]) {
+ const registryWithDatasets = withDatasets(registry);
+ const result = planCommit(
+ 'datasetName:"Support conversations"',
+ undefined,
+ registryWithDatasets,
+ );
+
+ expect(result.status).toBe("committed");
+ if (result.status !== "committed") continue;
+
+ expect(result.filters).toEqual([
+ {
+ column: "experimentDatasetId",
+ type: "stringOptions",
+ operator: "any of",
+ value: ["dataset-id-1"],
+ },
+ ]);
+ }
+ });
+
+ it("rejects unresolved dataset names instead of persisting them as IDs", () => {
+ const registry = withDatasets(EVALUATOR_FIELD_REGISTRY);
+ const result = planCommit(
+ 'datasetName:"Missing dataset"',
+ undefined,
+ registry,
+ );
+
+ expect(result.status).toBe("invalid");
+ if (result.status === "invalid") {
+ expect(result.diagnostics).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ message: '"Missing dataset" is not a valid dataset name',
+ }),
+ ]),
+ );
+ }
+ });
+
+ it("rejects partial-match operators for dataset names", () => {
+ const registry = withFieldOptions(EVALUATOR_FIELD_REGISTRY, "datasetName", [
+ { value: "dataset-id", displayValue: "Support" },
+ ]);
+
+ expect(
+ planCommit("datasetName:*Support*", undefined, registry).status,
+ ).toBe("invalid");
+ });
+
+ it("quotes dataset names containing grammar characters", () => {
+ expect(
+ filterStateToQueryText(
+ [
+ {
+ column: "experimentDatasetId",
+ type: "stringOptions",
+ operator: "any of",
+ value: ["dataset-id-2"],
+ },
+ ],
+ {},
+ withDatasets(EVALUATOR_FIELD_REGISTRY),
+ ).text,
+ ).toBe('datasetName:"Billing: escalations"');
+ });
+
+ it("keeps experimentDatasetName as a backward-compatible alias", () => {
+ const result = planCommit(
+ 'experimentDatasetName:"Support conversations"',
+ undefined,
+ withDatasets(EVALUATOR_FIELD_REGISTRY),
+ );
+
+ expect(result).toMatchObject({
+ status: "committed",
+ filters: [
+ {
+ column: "experimentDatasetId",
+ value: ["dataset-id-1"],
+ },
+ ],
+ });
+ });
+});
diff --git a/web/src/features/evals/v2/utils/datasetNameFilter.ts b/web/src/features/evals/v2/utils/datasetNameFilter.ts
new file mode 100644
index 0000000000..b79caf038d
--- /dev/null
+++ b/web/src/features/evals/v2/utils/datasetNameFilter.ts
@@ -0,0 +1,38 @@
+import type { ColumnDefinition } from "@langfuse/shared";
+
+import type { ObservedOptions } from "@/src/features/search-bar/lib/observed-options";
+
+export type DatasetFilterOption = {
+ id: string;
+ name: string;
+};
+
+export const DATASET_NAME_COLUMN = "datasetName";
+export const DATASET_NAME_FILTER_COLUMN: ColumnDefinition = {
+ name: "Dataset name",
+ id: DATASET_NAME_COLUMN,
+ aliases: [
+ "dataset",
+ "datasetname",
+ "dataset_name",
+ "experimentdatasetname",
+ "experiment_dataset_name",
+ ],
+ type: "stringOptions",
+ internal: "experiment_dataset_id",
+ options: [],
+ nullable: true,
+};
+
+export function addDatasetNameObservedOptions(
+ observed: ObservedOptions | undefined,
+ datasets: DatasetFilterOption[],
+): ObservedOptions | undefined {
+ if (observed === undefined) return undefined;
+ return {
+ ...observed,
+ [DATASET_NAME_COLUMN]: datasets.map((dataset) => ({
+ value: dataset.name,
+ })),
+ };
+}
diff --git a/web/src/features/search-bar/README.md b/web/src/features/search-bar/README.md
index 423a16dcdc..4807e78eb2 100644
--- a/web/src/features/search-bar/README.md
+++ b/web/src/features/search-bar/README.md
@@ -444,6 +444,11 @@ so a new view must check the field dropdown, not just that Enter applies.
3. **Reuse the view's `filterOptions` tRPC** for observed values —
`observed-options.ts` already maps that payload to per-column observed
values; point it at the new view's procedure (do not invent a parallel one).
+ When a field displays labels but persists stable values, declare its
+ canonical `filterColumn` in the registry overlay and hydrate it with
+ `withFieldOptions([{ value, displayValue }])`. The shared adapter then emits
+ the canonical column/value while the reverse adapter renders the label.
+ Do not add a host-specific post-lowering conversion.
4. **Keep the adapter targeting the shared `FilterState`.** Reuse the
already-registry-driven `operatorIssue`/`negationIssue` and the existing
per-kind lowering. Never add a second lowering path — that breaks the
diff --git a/web/src/features/search-bar/components/SearchComposer.tsx b/web/src/features/search-bar/components/SearchComposer.tsx
index 1f699ba3bc..6457ce2046 100644
--- a/web/src/features/search-bar/components/SearchComposer.tsx
+++ b/web/src/features/search-bar/components/SearchComposer.tsx
@@ -1515,7 +1515,7 @@ export function SearchComposer({
</div>
{showGlobalDiagnostics && (
- <div className="absolute top-1.5 right-2 flex items-center gap-1">
+ <div className="absolute top-1/2 right-2 flex -translate-y-1/2 items-center gap-1">
<span
className="text-destructive"
title={visibleDiagnostics.map((d) => d.message).join("; ")}
diff --git a/web/src/features/search-bar/hooks/useEventsSearchBar.ts b/web/src/features/search-bar/hooks/useEventsSearchBar.ts
index a3a6d46184..7e6489694e 100644
--- a/web/src/features/search-bar/hooks/useEventsSearchBar.ts
+++ b/web/src/features/search-bar/hooks/useEventsSearchBar.ts
@@ -111,11 +111,13 @@ export function useEventsSearchBar({
// validation so both route `scores.<name>` by the same observed score type.
const observedRef = useRef(observed);
observedRef.current = observed;
+ const registryRef = useRef(registry);
+ registryRef.current = registry;
const [store] = useState(() =>
createSearchBarStore(
() => scoreTypeContextFromObserved(observedRef.current),
- registry,
+ () => registryRef.current,
),
);
@@ -144,17 +146,14 @@ export function useEventsSearchBar({
store.getState().actions.resetTo(committedText);
}, [enabled, committedText, store]);
- // Re-validate when observed options load: a draft typed before score types
- // were known has a stale draftValid (the editor's red-border gate reads it),
- // so without this a `scores.<numeric>:<non-number>` typed during the load
- // window would commit-reject with no visible error. `observed` identity is
- // NOT stable across refetches (a relative range + auto-refresh rebuilds it
- // every tick), so this can fire on ticks where the score types are unchanged
- // — revalidate() bails on a set-equal context, keeping that path a no-op.
+ // Re-validate when observed options or the registry load: a draft typed
+ // before score types or dynamic allowed values were known has stale
+ // draftValid. Their identities can rotate across refetches, so revalidate()
+ // bails when both effective contexts are unchanged.
useEffect(() => {
if (!enabled) return;
store.getState().actions.revalidate();
- }, [enabled, observed, store]);
+ }, [enabled, observed, registry, store]);
// Latest applied-state setters, read inside commit without rebuilding it.
const applyRef = useRef({ setFilterState, setSearchQuery, setSearchType });
diff --git a/web/src/features/search-bar/lib/adapter.ts b/web/src/features/search-bar/lib/adapter.ts
index 920beed337..d5b6170ea1 100644
--- a/web/src/features/search-bar/lib/adapter.ts
+++ b/web/src/features/search-bar/lib/adapter.ts
@@ -468,11 +468,18 @@ function lowerText(
return;
}
if (field.syncMode === "exactOption") {
+ const values = field.filterValueByDisplayValue
+ ? node.values.map((value) => field.filterValueByDisplayValue!.get(value))
+ : node.values;
+ if (values.some((value) => value === undefined)) {
+ errors.push(`"${field.id}" contains an unknown option`);
+ return;
+ }
out.push({
type: "stringOptions",
- column: field.id,
+ column: field.filterColumn ?? field.id,
operator: negated ? "none of" : "any of",
- value: node.values,
+ value: values as string[],
});
return;
}
@@ -866,7 +873,7 @@ function lowerHas(
}
out.push({
type: "null",
- column: target.field.id,
+ column: target.field.filterColumn ?? target.field.id,
operator: negated ? "is null" : "is not null",
value: "",
});
diff --git a/web/src/features/search-bar/lib/fields.ts b/web/src/features/search-bar/lib/fields.ts
index 80b5226f80..9aebb02719 100644
--- a/web/src/features/search-bar/lib/fields.ts
+++ b/web/src/features/search-bar/lib/fields.ts
@@ -16,7 +16,11 @@
// AUTOCOMPLETE picker independently — `id`/`name` are textSearch (so `id:abc`
// is a substring search) yet keep their observed-value picker.
-import { eventsTableCols, type ColumnDefinition } from "@langfuse/shared";
+import {
+ eventsTableCols,
+ type ColumnDefinition,
+ type SingleValueOption,
+} from "@langfuse/shared";
import type { CompareOp } from "./ast";
import { quoteIfNeeded, unquote } from "./quoting";
@@ -25,7 +29,8 @@ type FieldKind = "text" | "number" | "datetime" | "boolean";
type SyncMode = "exactOption" | "arrayOption" | "textSearch";
export type FieldDef = {
- /** Canonical field id; also the filter `column` sent to the API. */
+ /** Canonical query field id; defaults to the FilterState column unless
+ * `filterColumn` maps a display-oriented field to canonical storage. */
id: string;
/** Lowercase aliases accepted by the grammar (canonical id always works too). */
aliases: string[];
@@ -52,12 +57,23 @@ export type FieldDef = {
* `exactOption`; only set this on `textSearch` fields that should suggest.
*/
suggestObservedValues?: boolean;
+ /** Canonical FilterState column emitted for this display-oriented field. */
+ filterColumn?: string;
+ /** Display/query value → canonical FilterState value for labeled options. */
+ filterValueByDisplayValue?: ReadonlyMap<string, string>;
+ /** Canonical FilterState value → display/query value for labeled options. */
+ displayValueByFilterValue?: ReadonlyMap<string, string>;
/** Nullable-only columns participate in `has:` but are not direct filters. */
directFilter?: boolean;
};
export type FieldRegistry = {
- id: "events" | "evaluationRules" | "sessions";
+ id:
+ | "events"
+ | "evaluationRules"
+ | "evaluatorSamples"
+ | "ruleSamples"
+ | "sessions";
fields: readonly FieldDef[];
columns: readonly ColumnDefinition[];
allowFreeText: boolean;
@@ -163,6 +179,9 @@ export function fieldRegistryFromColumns(
nullable: column.nullable,
directFilter: column.type !== "null",
suggestObservedValues: fieldOverlay?.suggestObservedValues,
+ filterColumn: fieldOverlay?.filterColumn,
+ filterValueByDisplayValue: fieldOverlay?.filterValueByDisplayValue,
+ displayValueByFilterValue: fieldOverlay?.displayValueByFilterValue,
unit: fieldOverlay?.unit,
negatedLabel: fieldOverlay?.negatedLabel,
};
@@ -184,6 +203,73 @@ export function fieldRegistryFromColumns(
});
}
+export function extendFieldRegistryWithColumns(
+ registry: FieldRegistry,
+ columns: readonly ColumnDefinition[],
+ fieldOverlays?: Readonly<Record<string, FieldOverlay>>,
+): FieldRegistry {
+ const addedFields = fieldRegistryFromColumns(columns, {
+ id: registry.id,
+ fields: fieldOverlays,
+ }).fields;
+
+ return createFieldRegistry({
+ id: registry.id,
+ fields: [...registry.fields, ...addedFields],
+ columns: [...registry.columns, ...columns],
+ metadata: registry.metadata,
+ scores: registry.scores,
+ traceScores: registry.traceScores,
+ allowFreeText: registry.allowFreeText,
+ defaultTextField: registry.defaultTextField,
+ searchExamples: registry.searchExamples,
+ recentSearches: registry.recentSearches,
+ aiFilterPrompt: registry.aiFilterPrompt,
+ aiContextFields: registry.aiContextFields,
+ });
+}
+
+export function withFieldOptions(
+ registry: FieldRegistry,
+ fieldId: string,
+ options: readonly SingleValueOption[],
+): FieldRegistry {
+ const filterValueByDisplayValue = new Map(
+ options.map((option) => [
+ option.displayValue ?? option.value,
+ option.value,
+ ]),
+ );
+ const displayValueByFilterValue = new Map(
+ options.map((option) => [
+ option.value,
+ option.displayValue ?? option.value,
+ ]),
+ );
+ return createFieldRegistry({
+ id: registry.id,
+ fields: registry.fields.map((field) =>
+ field.id === fieldId
+ ? {
+ ...field,
+ filterValueByDisplayValue,
+ displayValueByFilterValue,
+ }
+ : field,
+ ),
+ columns: registry.columns,
+ metadata: registry.metadata,
+ scores: registry.scores,
+ traceScores: registry.traceScores,
+ allowFreeText: registry.allowFreeText,
+ defaultTextField: registry.defaultTextField,
+ searchExamples: registry.searchExamples,
+ recentSearches: registry.recentSearches,
+ aiFilterPrompt: registry.aiFilterPrompt,
+ aiContextFields: registry.aiContextFields,
+ });
+}
+
// prettier-ignore
export const FIELDS: FieldDef[] = [
{ id: "id", aliases: ["spanid", "span_id", "observationid", "observation_id"], kind: "text", syncMode: "textSearch", suggestObservedValues: true, label: "Observation ID", description: "Observation/span identifier" },
@@ -318,6 +404,11 @@ function createFieldRegistry({
columnIds.set(alias.toLowerCase(), column.id);
}
}
+ for (const field of fields) {
+ if (field.filterColumn) {
+ columnIds.set(field.filterColumn.toLowerCase(), field.id);
+ }
+ }
const registry: FieldRegistry = {
id,
@@ -529,6 +620,9 @@ export function operatorIssue(
if (f.directFilter === false) {
return `"${f.id}" only supports presence checks (has:${f.id} or -has:${f.id})`;
}
+ if (f.filterColumn && (op === "~" || op === "^" || op === "$")) {
+ return `"${f.id}" maps labeled options to exact stored values and does not support ${label(op)}`;
+ }
if (f.kind === "number") {
if (op === "~" || op === "^" || op === "$") {
return `"${f.id}" is a number field and does not support ${label(op)}`;
diff --git a/web/src/features/search-bar/lib/filter-state-to-query.ts b/web/src/features/search-bar/lib/filter-state-to-query.ts
index e95cd4410c..f3fd91a22e 100644
--- a/web/src/features/search-bar/lib/filter-state-to-query.ts
+++ b/web/src/features/search-bar/lib/filter-state-to-query.ts
@@ -73,15 +73,31 @@ function lowerSingle(
switch (filter.type) {
case "stringOptions":
case "arrayOptions": {
- const id = registry.columnIdOf(filter.column);
+ let id = registry.columnIdOf(filter.column);
if (id === null || filter.value.length === 0) return null;
+ let values = filter.value;
+ let ref = registry.resolveField(id);
+ const displayValueByFilterValue =
+ ref?.type === "field" ? ref.field.displayValueByFilterValue : undefined;
+ if (displayValueByFilterValue !== undefined) {
+ const displayValues = filter.value.map((value) =>
+ displayValueByFilterValue.get(value),
+ );
+ if (displayValues.every((value) => value !== undefined)) {
+ values = displayValues as string[];
+ } else {
+ // A deleted option has no label mapping. Keep its canonical field and
+ // value so the filter remains visible and round-trips losslessly.
+ id = filter.column;
+ ref = registry.resolveField(id);
+ }
+ }
// A stringOptions/arrayOptions filter is EXACT-set semantics. On a
// `textSearch` field (id/name) the bar reads a bare single value as
// `contains`, so a single-value any-of/none-of would silently flip
// exact→substring on the next commit. The single-value forms therefore use
// the explicit exact operator (`name:=abc` / `-name:=abc`); the grouped
// multi-value forms already reparse to exact any-of/none-of.
- const ref = registry.resolveField(id);
const isTextSearch =
ref?.type === "field" && ref.field.syncMode === "textSearch";
if (filter.operator === "none of") {
@@ -90,9 +106,9 @@ function lowerSingle(
// none-of — NOT the bare `-name:abc`, which is does-not-contain
// (substring). The grouped/option forms use the bare `=` any-of shape.
if (isTextSearch && filter.value.length === 1) {
- return negate(filterNode(id, "exact", filter.value));
+ return negate(filterNode(id, "exact", values));
}
- return negate(filterNode(id, "=", filter.value));
+ return negate(filterNode(id, "=", values));
}
if (filter.operator === "all of") {
// A single-value all-of has no distinct grammar form — `(a)` reparses
@@ -100,18 +116,22 @@ function lowerSingle(
// the next commit. Skip it (preserved via skippedFilters) rather than
// rewrite; multi-value all-of serializes to the `(a AND b)` group.
if (filter.value.length < 2) return null;
- return filterNode(id, "=", filter.value, "and");
+ return filterNode(id, "=", values, "and");
}
// Single-value any-of on a textSearch field: emit the explicit exact form
// (`id:=abc`) so it round-trips to `{string,=}` (exact preserved), not the
// bare `id:abc` that would re-lower to `contains`.
if (isTextSearch && filter.value.length === 1) {
- return filterNode(id, "exact", filter.value);
+ return filterNode(id, "exact", values);
}
- return filterNode(id, "=", filter.value);
+ return filterNode(id, "=", values);
}
case "string": {
- const id = registry.columnIdOf(filter.column);
+ const directRef = registry.resolveField(filter.column);
+ const id =
+ directRef?.type === "field"
+ ? directRef.field.id
+ : registry.columnIdOf(filter.column);
if (id === null) return null;
if (filter.operator === "does not contain") {
// Mirror the positive contains carve-out below: a textSearch field emits
diff --git a/web/src/features/search-bar/lib/searchBarInvariants.clienttest.ts b/web/src/features/search-bar/lib/searchBarInvariants.clienttest.ts
index 118d93bb6e..be96432ba7 100644
--- a/web/src/features/search-bar/lib/searchBarInvariants.clienttest.ts
+++ b/web/src/features/search-bar/lib/searchBarInvariants.clienttest.ts
@@ -2,8 +2,16 @@
import { describe, expect, it } from "vitest";
-import { EVENTS_FIELD_REGISTRY } from "./fields";
+import {
+ EVENTS_FIELD_REGISTRY,
+ type FieldRegistry,
+ withFieldOptions,
+} from "./fields";
import { RULE_FIELD_REGISTRY } from "@/src/features/evals/v2/constants/ruleSearchRegistry";
+import {
+ EVALUATOR_FIELD_REGISTRY,
+ RULE_SAMPLE_FIELD_REGISTRY,
+} from "@/src/features/evals/v2/constants/evaluatorSearchRegistry";
import { SESSIONS_FIELD_REGISTRY } from "@/src/features/filters/config/sessionsSearchRegistry";
import { validateQuery } from "./validate";
import { planCommit } from "./commit";
@@ -99,6 +107,59 @@ const evaluationRulesView: RegistryUnderTest = {
freeTextValues: [],
};
+const withInvariantDatasetOptions = (
+ registry: FieldRegistry,
+ values: string[],
+) =>
+ withFieldOptions(
+ registry,
+ "datasetName",
+ values.map((displayValue, index) => ({
+ value: `dataset-${index}`,
+ displayValue,
+ })),
+ );
+
+const sampleFilterViews: RegistryUnderTest[] = [
+ {
+ ...eventsView,
+ name: "evaluator sample filters",
+ registry: withInvariantDatasetOptions(
+ EVALUATOR_FIELD_REGISTRY,
+ eventsView.fieldValues,
+ ),
+ },
+ {
+ ...evaluationRulesView,
+ name: "rule sample filters",
+ registry: withInvariantDatasetOptions(
+ RULE_SAMPLE_FIELD_REGISTRY,
+ evaluationRulesView.fieldValues,
+ ),
+ },
+];
+
+describe.each(sampleFilterViews)(
+ "search bar invariants — $name registry",
+ (view) => {
+ it("holds parity, round-trip, and serialization invariants", () => {
+ const failures = runSearchBarInvariants(view);
+ expect(
+ failures,
+ failures.length === 0
+ ? "ok"
+ : `\n${failures
+ .slice(0, 25)
+ .map(
+ (failure) =>
+ ` [${failure.invariant}] ${failure.case} — ${failure.detail}`,
+ )
+ .join("\n")}`,
+ ).toEqual([]);
+ });
+ },
+);
+
describe("search bar invariants — events v4 registry", () => {
it("generates a broad field × operator × value matrix", () => {
// Sanity: the matrix actually exercises the registry (guards against a
diff --git a/web/src/features/search-bar/lib/validate.ts b/web/src/features/search-bar/lib/validate.ts
index 96b614153b..0a4eb0e25b 100644
--- a/web/src/features/search-bar/lib/validate.ts
+++ b/web/src/features/search-bar/lib/validate.ts
@@ -159,6 +159,43 @@ function incompleteFieldTokenDiagnostics(
}
}
+function labeledOptionValueDiagnostics(
+ node: ASTNode,
+ textLength: number,
+ out: Diagnostic[],
+ registry: FieldRegistry,
+): void {
+ if (node.kind === "not") {
+ labeledOptionValueDiagnostics(node.child, textLength, out, registry);
+ return;
+ }
+ if (node.kind === "and" || node.kind === "or") {
+ for (const child of node.children) {
+ labeledOptionValueDiagnostics(child, textLength, out, registry);
+ }
+ return;
+ }
+ if (node.kind !== "filter") return;
+
+ const ref = registry.resolveField(node.key);
+ if (
+ ref?.type !== "field" ||
+ ref.field.filterValueByDisplayValue === undefined
+ )
+ return;
+
+ for (const value of node.values) {
+ if (ref.field.filterValueByDisplayValue.has(value)) continue;
+ const span = nodeSpan(node, textLength);
+ out.push({
+ from: span.from,
+ to: span.to,
+ severity: "error",
+ message: `"${value}" is not a valid ${ref.field.label.toLowerCase()}`,
+ });
+ }
+}
+
export function semanticDiagnostics(
ast: ASTNode | null,
textLength: number,
@@ -172,6 +209,7 @@ export function semanticDiagnostics(
// filter, not free text — checked over the WHOLE tree (not per top-level
// node) so the adjacency scoping can see each text node's siblings.
incompleteFieldTokenDiagnostics(ast, out, registry);
+ labeledOptionValueDiagnostics(ast, textLength, out, registry);
// Lower each top-level node independently so error spans point at the
// offending node instead of the whole query. The lowering must see the same
diff --git a/web/src/features/search-bar/server/buildFilterPrompt.ts b/web/src/features/search-bar/server/buildFilterPrompt.ts
index 1a3f24b39c..4e8c87f291 100644
--- a/web/src/features/search-bar/server/buildFilterPrompt.ts
+++ b/web/src/features/search-bar/server/buildFilterPrompt.ts
@@ -154,7 +154,7 @@ export function buildFilterSystemPrompt(
const catalog = buildFieldCatalog(registry);
const nullableIds = nullableFieldIds(registry);
- if (registry.id === "evaluationRules") {
+ if (registry.id === "evaluationRules" || registry.id === "ruleSamples") {
return `You generate observation filters for a Langfuse evaluation rule.
Respond with ONLY a JSON array of flat FilterState objects. Use only canonical
columns from this catalog and only the documented type/operator/value shape:
diff --git a/web/src/features/search-bar/server/parseFilterCompletion.ts b/web/src/features/search-bar/server/parseFilterCompletion.ts
index de5281c8b0..98a387fba4 100644
--- a/web/src/features/search-bar/server/parseFilterCompletion.ts
+++ b/web/src/features/search-bar/server/parseFilterCompletion.ts
@@ -11,6 +11,7 @@ import {
SCORE_COLUMNS,
type FieldRegistry,
} from "../lib/fields";
+import { planCommit } from "../lib/commit";
import { filterStateToQueryText } from "../lib/filter-state-to-query";
import type { ObservedScoreNames } from "../lib/observed-options";
@@ -287,15 +288,32 @@ export function parseGeneratedFilters(
compatible,
scoreNames,
);
+ const projected: FilterState = [];
+ for (const filter of scoreChecked) {
+ const id = registry.columnIdOf(filter.column);
+ const ref = id === null ? null : registry.resolveField(id);
+ if (ref?.type !== "field" || ref.field.filterColumn === undefined) {
+ projected.push(filter);
+ continue;
+ }
+
+ const rendered = filterStateToQueryText([filter], {}, registry);
+ const result = planCommit(rendered.text, undefined, registry);
+ if (rendered.skippedFilters.length === 0 && result.status === "committed") {
+ for (const canonicalFilter of result.filters) {
+ projected.push(canonicalFilter);
+ }
+ }
+ }
// Guardrail 3: drop anything that doesn't round-trip to bar grammar (unknown /
// non-representable columns land in skippedFilters).
const { text, skippedFilters } = filterStateToQueryText(
- scoreChecked,
+ projected,
{},
registry,
);
const skipped = new Set(skippedFilters);
- const filters = scoreChecked.filter((f) => !skipped.has(f));
+ const filters = projected.filter((f) => !skipped.has(f));
return {
filters,
queryText: text,
diff --git a/web/src/features/search-bar/server/router.ts b/web/src/features/search-bar/server/router.ts
index 792d010fd3..1702075a9e 100644
--- a/web/src/features/search-bar/server/router.ts
+++ b/web/src/features/search-bar/server/router.ts
@@ -49,8 +49,13 @@ import {
} from "./parseOutcomeScoring";
import { getProductBaseUrl } from "@/src/utils/base-url";
import { RULE_FIELD_REGISTRY } from "@/src/features/evals/v2/constants/ruleSearchRegistry";
+import {
+ EVALUATOR_FIELD_REGISTRY,
+ RULE_SAMPLE_FIELD_REGISTRY,
+} from "@/src/features/evals/v2/constants/evaluatorSearchRegistry";
+import { DATASET_NAME_COLUMN } from "@/src/features/evals/v2/utils/datasetNameFilter";
import { SESSIONS_FIELD_REGISTRY } from "@/src/features/filters/config/sessionsSearchRegistry";
-import { EVENTS_FIELD_REGISTRY } from "../lib/fields";
+import { EVENTS_FIELD_REGISTRY, withFieldOptions } from "../lib/fields";
// Caps shared with `observedScoreNamesFromOptions` (the client-side builder),
// which sends a set as undefined instead of ever exceeding them.
@@ -62,7 +67,13 @@ const GenerateFilterInput = z.object({
projectId: z.string(),
prompt: z.string().min(1).max(2048),
registryId: z
- .enum(["events", "evaluationRules", "sessions"])
+ .enum([
+ "events",
+ "evaluationRules",
+ "evaluatorSamples",
+ "ruleSamples",
+ "sessions",
+ ])
.default("events"),
/** Existing bar query text, so the model refines the current filters. */
currentQuery: z.string().max(4096).optional(),
@@ -89,16 +100,20 @@ export const searchBarRouter = createTRPCRouter({
.input(GenerateFilterInput)
.mutation(async ({ input, ctx }) => {
try {
- const registry =
+ const baseRegistry =
input.registryId === "evaluationRules"
? RULE_FIELD_REGISTRY
- : input.registryId === "sessions"
- ? SESSIONS_FIELD_REGISTRY
- : EVENTS_FIELD_REGISTRY;
+ : input.registryId === "evaluatorSamples"
+ ? EVALUATOR_FIELD_REGISTRY
+ : input.registryId === "ruleSamples"
+ ? RULE_SAMPLE_FIELD_REGISTRY
+ : input.registryId === "sessions"
+ ? SESSIONS_FIELD_REGISTRY
+ : EVENTS_FIELD_REGISTRY;
// Defence in depth for the client gate above: generating against the
// fallback (events) prompt for a view that has no branch of its own
// produces filters for columns that view does not have.
- if (!registry.aiFilterPrompt) {
+ if (!baseRegistry.aiFilterPrompt) {
throw new TRPCError({
code: "BAD_REQUEST",
message: "AI filter generation is not available for this view",
@@ -113,14 +128,25 @@ export const searchBarRouter = createTRPCRouter({
scope: "project:read",
});
- const project = await ctx.prisma.project.findUnique({
- where: { id: input.projectId },
- select: {
- organization: {
- select: { aiFeaturesEnabled: true, aiTelemetryEnabled: true },
+ const isSampleRegistry =
+ input.registryId === "evaluatorSamples" ||
+ input.registryId === "ruleSamples";
+ const [project, datasets] = await Promise.all([
+ ctx.prisma.project.findUnique({
+ where: { id: input.projectId },
+ select: {
+ organization: {
+ select: { aiFeaturesEnabled: true, aiTelemetryEnabled: true },
+ },
},
- },
- });
+ }),
+ isSampleRegistry
+ ? ctx.prisma.dataset.findMany({
+ where: { projectId: input.projectId },
+ select: { id: true, name: true },
+ })
+ : Promise.resolve([]),
+ ]);
if (!project) {
throw new TRPCError({
@@ -136,6 +162,17 @@ export const searchBarRouter = createTRPCRouter({
});
}
+ const registry = isSampleRegistry
+ ? withFieldOptions(
+ baseRegistry,
+ DATASET_NAME_COLUMN,
+ datasets.map((dataset) => ({
+ value: dataset.id,
+ displayValue: dataset.name,
+ })),
+ )
+ : baseRegistry;
+
const modelConfig = getInAppAgentModelConfig();
if (!modelConfig) {
diff --git a/web/src/features/search-bar/store/searchBarStore.clienttest.ts b/web/src/features/search-bar/store/searchBarStore.clienttest.ts
index f4d76c564a..4a9cda8fba 100644
--- a/web/src/features/search-bar/store/searchBarStore.clienttest.ts
+++ b/web/src/features/search-bar/store/searchBarStore.clienttest.ts
@@ -1,4 +1,6 @@
import { createSearchBarStore } from "@/src/features/search-bar/store/searchBarStore";
+import { EVALUATOR_FIELD_REGISTRY } from "@/src/features/evals/v2/constants/evaluatorSearchRegistry";
+import { withFieldOptions } from "@/src/features/search-bar/lib/fields";
describe("searchBarStore (draft-only)", () => {
it("validates the draft on setDraft", () => {
@@ -242,6 +244,24 @@ describe("searchBarStore (draft-only)", () => {
expect(store.getState().draftValid).toBe(false);
});
+ it("revalidates drafts against the latest dynamic registry", () => {
+ let registry = withFieldOptions(
+ EVALUATOR_FIELD_REGISTRY,
+ "datasetName",
+ [],
+ );
+ const store = createSearchBarStore(undefined, () => registry);
+ store.getState().actions.setDraft('datasetName:"Filter QA Dataset"');
+ expect(store.getState().draftValid).toBe(false);
+
+ registry = withFieldOptions(EVALUATOR_FIELD_REGISTRY, "datasetName", [
+ { value: "dataset-id", displayValue: "Filter QA Dataset" },
+ ]);
+ store.getState().actions.revalidate();
+
+ expect(store.getState().draftValid).toBe(true);
+ });
+
it("revalidate reveals the red state when a valid draft flips invalid", () => {
// Committed against an empty context while filterOptions loaded; once score
// types arrive and unmask it, the bar must paint red (not stay silently
diff --git a/web/src/features/search-bar/store/searchBarStore.ts b/web/src/features/search-bar/store/searchBarStore.ts
index 9061632ea0..dbd5ffaf1f 100644
--- a/web/src/features/search-bar/store/searchBarStore.ts
+++ b/web/src/features/search-bar/store/searchBarStore.ts
@@ -99,26 +99,35 @@ export type SearchBarStoreState = {
export type SearchBarStore = StoreApi<SearchBarStoreState>;
/**
- * `resolveScoreTypes` lets draft validation route `scores.<name>` by observed
- * score type — the SAME context planCommit uses — so the store's draftValid
- * (which the editor's red-border gate reads) never disagrees with the commit
- * gate. A thunk so it always reads the latest observed options.
+ * The resolver thunks keep draft validation aligned with the same live score
+ * context and field registry that planCommit uses. This matters for registries
+ * whose allowed values arrive asynchronously.
*/
export function createSearchBarStore(
resolveScoreTypes?: () => ScoreTypeContext | undefined,
- registry: FieldRegistry = EVENTS_FIELD_REGISTRY,
+ registryOrResolver:
+ | FieldRegistry
+ | (() => FieldRegistry) = EVENTS_FIELD_REGISTRY,
): SearchBarStore {
+ const resolveRegistry =
+ typeof registryOrResolver === "function"
+ ? registryOrResolver
+ : () => registryOrResolver;
+
return createStore<SearchBarStoreState>((set, get) => {
// The score-type context used by the most recent validation. revalidate()
// bails when the context is set-equal to this, so observed-identity churn
// (a relative range + auto-refresh rebuilds the context every tick) doesn't
// re-parse the draft and emit a fresh diagnostics array for no change.
let lastScoreTypes: ScoreTypeContext | undefined;
+ let lastRegistry: FieldRegistry | undefined;
let hasValidated = false;
const writeDraft = (next: string) => {
const scoreTypes = resolveScoreTypes?.();
+ const registry = resolveRegistry();
lastScoreTypes = scoreTypes;
+ lastRegistry = registry;
hasValidated = true;
const res = validateQuery(next, scoreTypes, registry);
set({
@@ -159,6 +168,7 @@ export function createSearchBarStore(
// Number-normalize numeric (not categorical) score values. It preserves
// structure/order, so free-text canonicalization is kept.
const scoreTypes = resolveScoreTypes?.();
+ const registry = resolveRegistry();
if (
draftsSemanticallyEqual(committedText, draft, scoreTypes, registry)
)
@@ -166,6 +176,7 @@ export function createSearchBarStore(
writeDraft(committedText);
},
removeChipSpan: (from, to) => {
+ const registry = resolveRegistry();
const next = removeToken(get().draft, { from, to }, registry);
writeDraft(next);
return next;
@@ -181,13 +192,19 @@ export function createSearchBarStore(
},
revalidate: () => {
const scoreTypes = resolveScoreTypes?.();
+ const registry = resolveRegistry();
// Nothing to refresh: the draft was already validated against an
- // equal context (writeDraft and revalidate both record it), so the
- // result would be identical. Skip the parse + set to avoid a
- // no-op re-render from the fresh diagnostics array.
- if (hasValidated && scoreTypeContextEqual(scoreTypes, lastScoreTypes))
+ // equal context and registry (writeDraft and revalidate both record
+ // them), so the result would be identical. Skip the parse + set to
+ // avoid a no-op re-render from the fresh diagnostics array.
+ if (
+ hasValidated &&
+ registry === lastRegistry &&
+ scoreTypeContextEqual(scoreTypes, lastScoreTypes)
+ )
return;
lastScoreTypes = scoreTypes;
+ lastRegistry = registry;
hasValidated = true;
const wasValid = get().draftValid;
const res = validateQuery(get().draft, scoreTypes, registry);