Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(evals): add observation filter builder (#16930)

Tobias Wochinger858377d0a084
diff --git a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/SampleObservationSelectorBase.tsx b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/SampleObservationSelectorBase.tsx
index e20bdd8101..ed7a6b48cc 100644
--- a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/SampleObservationSelectorBase.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/SampleObservationSelectorBase.tsx
@@ -15,36 +15,33 @@ import { useRowHeightLocalStorage } from "@/src/components/table/data-table-row-
import type { LangfuseColumnDef } from "@/src/components/table/types";
import { Button } from "@/src/components/ui/button";
import { Skeleton } from "@/src/components/ui/skeleton";
-import { useEventsFilterOptions } from "@/src/features/events/hooks/useEventsFilterOptions";
+import useLocalStorage from "@/src/components/useLocalStorage";
import { EventsSearchBarRow } from "@/src/features/search-bar/components/EventsSearchBarRow";
import { useEventsSearchBar } from "@/src/features/search-bar/hooks/useEventsSearchBar";
import { buildAiContext } from "@/src/features/search-bar/lib/ai-context";
import {
type FieldRegistry,
EVENTS_FIELD_REGISTRY,
- withFieldOptions,
} from "@/src/features/search-bar/lib/fields";
-import {
- observedScoreNamesFromOptions,
- toObservedOptions,
-} from "@/src/features/search-bar/lib/observed-options";
+import { observedScoreNamesFromOptions } from "@/src/features/search-bar/lib/observed-options";
import useColumnOrder from "@/src/features/column-visibility/hooks/useColumnOrder";
import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
import { api, sendAsPostOption, type RouterOutputs } from "@/src/utils/api";
import type { AbsoluteTimeRange } from "@/src/utils/date-range-utils";
import { SectionHeader } from "@/src/features/evals/v2/components/Evaluators/Testing/components/SectionHeader/SectionHeader";
+import { EVALUATOR_FILTER_EXPERIENCE_STORAGE_KEY } from "@/src/features/evals/v2/constants/evaluatorFilterExperience";
import { EXPERIMENTS_AND_EVALS_EXCLUSION_FILTERS } from "@/src/features/evals/v2/constants/experimentAndEvalFilters";
-import {
- buildSampleQueryFilters,
- removeInternalEvaluationEnvironmentOptions,
-} from "@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/buildSampleQueryFilters";
+import { FilterModeToggle } from "@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/components/FilterModeToggle";
+import { ObservationFilterBuilder } from "@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/components/ObservationFilterBuilder/ObservationFilterBuilder";
+import { buildSampleQueryFilters } from "@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/buildSampleQueryFilters";
import { dedupeObservationPages } from "@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/dedupeObservations";
import { toggleExampleFilters } from "@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/toggleExampleFilters";
import {
- DATASET_NAME_COLUMN,
- addDatasetNameObservedOptions,
-} from "@/src/features/evals/v2/utils/datasetNameFilter";
+ type MapSampleObservedOptions,
+ useSampleObservationFilterOptions,
+} from "@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/hooks/useSampleObservationFilterOptions";
import { useReusableRuleFilterPresets } from "@/src/features/evals/v2/hooks/useReusableRuleFilterPresets";
+import type { EvaluatorFilterExperience } from "@/src/features/evals/v2/types/evaluatorFilterExperience";
import { usePostHogClientCapture } from "@/src/features/posthog-analytics";
export type SampleObservation =
@@ -152,9 +149,7 @@ export type SampleObservationSelectorBaseProps = {
matchingDescription: string;
matchingTooltip: string;
formatCount: (count: number) => string;
- mapObservedOptions: (
- observed: ReturnType<typeof toObservedOptions>,
- ) => ReturnType<typeof toObservedOptions>;
+ mapObservedOptions: MapSampleObservedOptions;
};
export function SampleObservationSelectorBase(
@@ -182,6 +177,11 @@ export function SampleObservationSelectorBase(
mapObservedOptions,
} = props;
const activeRegistry = registry ?? EVENTS_FIELD_REGISTRY;
+ const [filterMode, setFilterMode] =
+ useLocalStorage<EvaluatorFilterExperience>(
+ EVALUATOR_FILTER_EXPERIENCE_STORAGE_KEY,
+ "builder",
+ );
const datasets = api.datasets.allDatasetMeta.useQuery(
{ projectId },
{
@@ -192,39 +192,6 @@ export function SampleObservationSelectorBase(
},
);
const datasetOptions = useMemo(() => datasets.data ?? [], [datasets.data]);
- const searchRegistry = useMemo(
- () =>
- withFieldOptions(
- activeRegistry,
- DATASET_NAME_COLUMN,
- datasetOptions.map((dataset) => ({
- value: dataset.id,
- displayValue: dataset.name,
- })),
- ),
- [activeRegistry, datasetOptions],
- );
- const reusableRuleFilters = useReusableRuleFilterPresets(
- projectId,
- searchRegistry,
- );
- const capture = usePostHogClientCapture();
- const onQueryPresetPick = useCallback(
- (presetId: string) => {
- const preset = reusableRuleFilters.presets.find(
- (candidate) => candidate.id === presetId,
- );
- if (!preset) return;
- capture("evaluation_rules:filter_reused", {
- tableName,
- evaluatorCount: preset.evaluatorCount,
- filterCount: preset.filterCount,
- replacedFilterCount: filterState.length,
- isV4: true,
- });
- },
- [capture, filterState.length, reusableRuleFilters.presets, tableName],
- );
const setFilters = (
next: FilterState | ((current: FilterState) => FilterState),
) => {
@@ -252,25 +219,38 @@ export function SampleObservationSelectorBase(
() => buildSampleQueryFilters(previewFilters),
[previewFilters],
);
- const options = useEventsFilterOptions({
+ const options = useSampleObservationFilterOptions({
projectId,
startTimeFilter,
refiningFilter,
- includeApproxCount: true,
- lazy: true,
- });
- const observed = useMemo(() => {
- const visibleOptions = removeInternalEvaluationEnvironmentOptions(
- toObservedOptions(options.filterOptions, options.isFilterOptionsPending),
- );
- const mapped = mapObservedOptions(visibleOptions);
- return addDatasetNameObservedOptions(mapped, datasetOptions);
- }, [
+ filterMode,
+ activeRegistry,
datasetOptions,
mapObservedOptions,
- options.filterOptions,
- options.isFilterOptionsPending,
- ]);
+ });
+ const { searchRegistry, observed, builderColumns, queryOnlyColumnIds } =
+ options;
+ const reusableRuleFilters = useReusableRuleFilterPresets(
+ projectId,
+ searchRegistry,
+ );
+ const capture = usePostHogClientCapture();
+ const onQueryPresetPick = useCallback(
+ (presetId: string) => {
+ const preset = reusableRuleFilters.presets.find(
+ (candidate) => candidate.id === presetId,
+ );
+ if (!preset) return;
+ capture("evaluation_rules:filter_reused", {
+ tableName,
+ evaluatorCount: preset.evaluatorCount,
+ filterCount: preset.filterCount,
+ replacedFilterCount: filterState.length,
+ isV4: true,
+ });
+ },
+ [capture, filterState.length, reusableRuleFilters.presets, tableName],
+ );
const search = useEventsSearchBar({
projectId,
tableName,
@@ -498,11 +478,21 @@ export function SampleObservationSelectorBase(
meta={null}
description={filterDescription}
tooltip={filterTooltip}
- trailing={null}
+ trailing={
+ <FilterModeToggle
+ mode={filterMode}
+ onChange={(mode) => {
+ if (mode === "builder" && searchQuery !== null) {
+ search.applyFilters(filterState);
+ }
+ setFilterMode(mode);
+ }}
+ />
+ }
/>
{datasets.isPending ? (
<Skeleton className="h-10 w-full" />
- ) : (
+ ) : filterMode === "query" ? (
<EventsSearchBarRow
projectId={projectId}
tableName={tableName}
@@ -519,26 +509,35 @@ export function SampleObservationSelectorBase(
aiScoreNames={aiScoreNames}
className="p-0"
/>
+ ) : (
+ <ObservationFilterBuilder
+ columns={builderColumns}
+ filterState={filterState}
+ onChange={setFilters}
+ queryOnlyColumnIds={queryOnlyColumnIds}
+ />
)}
- <div className="flex flex-wrap gap-2">
- {EXAMPLES.map((example) => (
- <Button
- key={example.label}
- type="button"
- variant="outline"
- size="sm"
- className="flex h-8 items-center gap-2 text-sm"
- onClick={() => {
- setFilters((current) =>
- toggleExampleFilters(current, [...example.filters]),
- );
- }}
- >
- <example.icon className="h-4 w-4" />
- <span>{example.label}</span>
- </Button>
- ))}
- </div>
+ {filterMode === "query" ? (
+ <div className="flex flex-wrap gap-2">
+ {EXAMPLES.map((example) => (
+ <Button
+ key={example.label}
+ type="button"
+ variant="outline"
+ size="sm"
+ className="flex h-8 items-center gap-2 text-sm"
+ onClick={() => {
+ setFilters((current) =>
+ toggleExampleFilters(current, [...example.filters]),
+ );
+ }}
+ >
+ <example.icon className="h-4 w-4" />
+ <span>{example.label}</span>
+ </Button>
+ ))}
+ </div>
+ ) : null}
</section>
<section className="flex min-h-0 flex-col gap-2">
diff --git a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/components/FilterModeToggle.tsx b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/components/FilterModeToggle.tsx
new file mode 100644
index 0000000000..36b363a4f5
--- /dev/null
+++ b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/components/FilterModeToggle.tsx
@@ -0,0 +1,40 @@
+import { Braces, ListFilter } from "lucide-react";
+
+import type { EvaluatorFilterExperience } from "@/src/features/evals/v2/types/evaluatorFilterExperience";
+import { cn } from "@/src/utils/tailwind";
+
+export function FilterModeToggle({
+ mode,
+ onChange,
+}: {
+ mode: EvaluatorFilterExperience;
+ onChange: (mode: EvaluatorFilterExperience) => void;
+}) {
+ return (
+ <div
+ className="bg-muted flex items-center rounded-md p-0.5"
+ aria-label="Filter editor mode"
+ >
+ {(
+ [
+ { id: "query", label: "Query", icon: Braces },
+ { id: "builder", label: "Builder", icon: ListFilter },
+ ] as const
+ ).map((option) => (
+ <button
+ key={option.id}
+ type="button"
+ aria-pressed={mode === option.id}
+ onClick={() => onChange(option.id)}
+ className={cn(
+ "text-muted-foreground flex h-7 items-center gap-1 rounded px-2 text-xs",
+ mode === option.id && "bg-background text-foreground shadow-sm",
+ )}
+ >
+ <option.icon className="h-3.5 w-3.5" aria-hidden="true" />
+ {option.label}
+ </button>
+ ))}
+ </div>
+ );
+}
diff --git a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/components/ObservationFilterBuilder/ObservationFilterBuilder.clienttest.tsx b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/components/ObservationFilterBuilder/ObservationFilterBuilder.clienttest.tsx
new file mode 100644
index 0000000000..0facfca97e
--- /dev/null
+++ b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/components/ObservationFilterBuilder/ObservationFilterBuilder.clienttest.tsx
@@ -0,0 +1,62 @@
+import { render, screen } from "@testing-library/react";
+import type { ColumnDefinition, FilterState } from "@langfuse/shared";
+
+import { ObservationFilterBuilder } from "./ObservationFilterBuilder";
+
+let containerWidth = 560;
+
+vi.mock("@/src/hooks/useElementSize", () => ({
+ useElementSize: () => [
+ { current: null },
+ { width: containerWidth, height: 200 },
+ ],
+}));
+
+const columns: ColumnDefinition[] = [
+ {
+ id: "environment",
+ name: "Environment",
+ type: "stringOptions",
+ internal: "environment",
+ options: [{ value: "production" }],
+ },
+];
+
+const filterState: FilterState = [
+ {
+ column: "environment",
+ type: "stringOptions",
+ operator: "any of",
+ value: ["production"],
+ },
+];
+
+const renderBuilder = () =>
+ render(
+ <ObservationFilterBuilder
+ columns={columns}
+ filterState={filterState}
+ onChange={vi.fn()}
+ queryOnlyColumnIds={[]}
+ />,
+ );
+
+describe("ObservationFilterBuilder", () => {
+ beforeEach(() => {
+ containerWidth = 560;
+ });
+
+ it("keeps the table layout when space is limited", () => {
+ renderBuilder();
+
+ expect(screen.getByRole("table")).toBeInTheDocument();
+ });
+
+ it("wraps into the compact row layout when space is too narrow", () => {
+ containerWidth = 420;
+ renderBuilder();
+
+ expect(screen.queryByRole("table")).not.toBeInTheDocument();
+ expect(screen.getByText("Where")).toBeInTheDocument();
+ });
+});
diff --git a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/components/ObservationFilterBuilder/ObservationFilterBuilder.stories.tsx b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/components/ObservationFilterBuilder/ObservationFilterBuilder.stories.tsx
new file mode 100644
index 0000000000..2478a28220
--- /dev/null
+++ b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/components/ObservationFilterBuilder/ObservationFilterBuilder.stories.tsx
@@ -0,0 +1,82 @@
+import {
+ eventsEvalFilterColumns,
+ type ColumnDefinition,
+ type FilterState,
+} from "@langfuse/shared";
+import { fn } from "storybook/test";
+
+import preview from "../../../../../../../../../../../.storybook/preview";
+import { ObservationFilterBuilder } from "./ObservationFilterBuilder";
+
+const columns: ColumnDefinition[] = eventsEvalFilterColumns.map((column) => {
+ if (column.id !== "environment" || column.type !== "stringOptions") {
+ return column;
+ }
+ return {
+ ...column,
+ options: [{ value: "production" }, { value: "sdk-experiment" }],
+ };
+});
+
+const filterState = [
+ {
+ column: "isRootObservation",
+ type: "boolean",
+ operator: "=",
+ value: true,
+ },
+ {
+ column: "environment",
+ type: "stringOptions",
+ operator: "any of",
+ value: ["production"],
+ },
+ {
+ column: "type",
+ type: "stringOptions",
+ operator: "any of",
+ value: ["GENERATION"],
+ },
+ {
+ column: "experimentId",
+ type: "null",
+ operator: "is null",
+ value: "",
+ },
+] satisfies FilterState;
+
+const meta = preview.meta({ component: ObservationFilterBuilder });
+
+const args = {
+ columns,
+ filterState,
+ onChange: fn(),
+ queryOnlyColumnIds: ["metadata"],
+};
+
+export const Wide = meta.story({
+ args,
+ render: (storyArgs) => (
+ <div className="max-w-5xl">
+ <ObservationFilterBuilder {...storyArgs} />
+ </div>
+ ),
+});
+
+export const Narrow = meta.story({
+ args,
+ render: (storyArgs) => (
+ <div className="w-[560px] max-w-full">
+ <ObservationFilterBuilder {...storyArgs} />
+ </div>
+ ),
+});
+
+export const Compact = meta.story({
+ args,
+ render: (storyArgs) => (
+ <div className="w-[420px] max-w-full">
+ <ObservationFilterBuilder {...storyArgs} />
+ </div>
+ ),
+});
diff --git a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/components/ObservationFilterBuilder/ObservationFilterBuilder.tsx b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/components/ObservationFilterBuilder/ObservationFilterBuilder.tsx
new file mode 100644
index 0000000000..4e08e7d43a
--- /dev/null
+++ b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/components/ObservationFilterBuilder/ObservationFilterBuilder.tsx
@@ -0,0 +1,50 @@
+import type { ColumnDefinition, FilterState } from "@langfuse/shared";
+
+import { InlineFilterBuilder } from "@/src/features/filters";
+import { useElementSize } from "@/src/hooks/useElementSize";
+import { cn } from "@/src/utils/tailwind";
+
+export function ObservationFilterBuilder({
+ columns,
+ filterState,
+ onChange,
+ queryOnlyColumnIds,
+}: {
+ columns: ColumnDefinition[];
+ filterState: FilterState;
+ onChange: (filters: FilterState) => void;
+ queryOnlyColumnIds: string[];
+}) {
+ const [containerRef, containerSize] = useElementSize<HTMLDivElement>();
+ const containerWidth = containerSize?.width;
+ const isNarrow = containerWidth !== undefined && containerWidth < 720;
+ const isCompact = containerWidth !== undefined && containerWidth < 480;
+
+ return (
+ <div
+ ref={containerRef}
+ className={cn(
+ "[&_table]:w-full [&_table]:table-fixed [&_td:first-child]:w-16 [&_td:last-child]:w-8 [&_td:last-child]:text-right",
+ isNarrow
+ ? "[&_td:nth-child(2)]:w-40 [&_td:nth-child(3)]:w-32"
+ : "[&_td:nth-child(2)]:w-56 [&_td:nth-child(3)]:w-40",
+ )}
+ >
+ <InlineFilterBuilder
+ columns={columns}
+ filterState={filterState}
+ onChange={onChange}
+ columnIdentifier="id"
+ columnsHiddenUnlessSelected={queryOnlyColumnIds}
+ compact={isCompact}
+ subtleAddButton
+ columnsWithCustomSelect={[
+ "tags",
+ "name",
+ "traceName",
+ "calledToolNames",
+ ]}
+ />
+ </div>
+ );
+}
diff --git a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/buildSampleQueryFilters.clienttest.ts b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/buildSampleQueryFilters.clienttest.ts
index 977ae05279..1e856c6947 100644
--- a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/buildSampleQueryFilters.clienttest.ts
+++ b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/buildSampleQueryFilters.clienttest.ts
@@ -1,8 +1,9 @@
-import type { FilterState } from "@langfuse/shared";
+import type { ColumnDefinition, FilterState } from "@langfuse/shared";
import { describe, expect, it } from "vitest";
import {
buildSampleQueryFilters,
+ removeInternalEvaluationEnvironmentColumnOptions,
removeInternalEvaluationEnvironmentOptions,
} from "./buildSampleQueryFilters";
@@ -39,6 +40,25 @@ describe("buildSampleQueryFilters", () => {
expect(visibleFilters).toHaveLength(1);
});
+ it("hides internal evaluation environments from builder columns", () => {
+ const columns: ColumnDefinition[] = [
+ {
+ id: "environment",
+ name: "Environment",
+ type: "stringOptions",
+ internal: "environment",
+ options: [{ value: "langfuse-code-eval" }, { value: "production" }],
+ },
+ ];
+
+ expect(removeInternalEvaluationEnvironmentColumnOptions(columns)).toEqual([
+ {
+ ...columns[0],
+ options: [{ value: "production" }],
+ },
+ ]);
+ });
+
it("keeps valid experiment and user environment options visible", () => {
expect(
removeInternalEvaluationEnvironmentOptions({
diff --git a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/buildSampleQueryFilters.ts b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/buildSampleQueryFilters.ts
index 6286c082bc..49f93ccbf5 100644
--- a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/buildSampleQueryFilters.ts
+++ b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/buildSampleQueryFilters.ts
@@ -1,4 +1,4 @@
-import type { FilterState } from "@langfuse/shared";
+import type { ColumnDefinition, FilterState } from "@langfuse/shared";
import type { ObservedOptions } from "@/src/features/search-bar/lib/observed-options";
import {
@@ -17,6 +17,25 @@ export function buildSampleQueryFilters(
];
}
+export function removeInternalEvaluationEnvironmentColumnOptions(
+ columns: ColumnDefinition[],
+): ColumnDefinition[] {
+ const internalEnvironments = new Set<string>(
+ INTERNAL_EVALUATION_ENVIRONMENTS,
+ );
+ return columns.map((column) => {
+ if (column.id !== "environment" || column.type !== "stringOptions") {
+ return column;
+ }
+ return {
+ ...column,
+ options: column.options.filter(
+ ({ value }) => !internalEnvironments.has(value),
+ ),
+ };
+ });
+}
+
export function removeInternalEvaluationEnvironmentOptions(
observed: ObservedOptions | undefined,
): ObservedOptions | undefined {
diff --git a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/hooks/useSampleObservationFilterOptions.clienttest.ts b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/hooks/useSampleObservationFilterOptions.clienttest.ts
new file mode 100644
index 0000000000..2d137ff309
--- /dev/null
+++ b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/hooks/useSampleObservationFilterOptions.clienttest.ts
@@ -0,0 +1,82 @@
+import { renderHook } from "@testing-library/react";
+
+import { EVALUATOR_FIELD_REGISTRY } from "@/src/features/evals/v2/constants/evaluatorSearchRegistry";
+import { useSampleObservationFilterOptions } from "./useSampleObservationFilterOptions";
+
+const { useEventsFilterOptionsMock } = vi.hoisted(() => ({
+ useEventsFilterOptionsMock: vi.fn(),
+}));
+
+vi.mock("@/src/features/events/hooks/useEventsFilterOptions", () => ({
+ useEventsFilterOptions: useEventsFilterOptionsMock,
+}));
+
+const refiningFilter = [
+ {
+ column: "environment",
+ type: "stringOptions" as const,
+ operator: "any of" as const,
+ value: ["production"],
+ },
+];
+
+describe("useSampleObservationFilterOptions", () => {
+ beforeEach(() => {
+ useEventsFilterOptionsMock.mockReturnValue({
+ filterOptions: {
+ environment: [{ value: "langfuse-code-eval" }, { value: "production" }],
+ traceTags: [{ value: "customer-facing" }],
+ },
+ isFilterOptionsPending: false,
+ });
+ });
+
+ it("derives query and builder values from the same filter options", () => {
+ const { result } = renderHook(() =>
+ useSampleObservationFilterOptions({
+ projectId: "project-id",
+ startTimeFilter: [],
+ refiningFilter,
+ filterMode: "builder",
+ datasetOptions: [{ id: "dataset-id", name: "Support tickets" }],
+ mapObservedOptions: (observed) => observed,
+ activeRegistry: EVALUATOR_FIELD_REGISTRY,
+ }),
+ );
+
+ expect(useEventsFilterOptionsMock).toHaveBeenCalledWith(
+ expect.objectContaining({
+ refiningFilter,
+ lazy: false,
+ columns: expect.arrayContaining(["environment", "traceTags"]),
+ }),
+ );
+ expect(
+ result.current.searchRegistry.fields
+ .find((field) => field.id === "datasetName")
+ ?.displayValueByFilterValue?.get("dataset-id"),
+ ).toBe("Support tickets");
+ expect(result.current.observed?.environment).toEqual([
+ { value: "production" },
+ ]);
+ expect(result.current.observed?.datasetName).toEqual([
+ { value: "Support tickets" },
+ ]);
+
+ expect(
+ result.current.builderColumns.find(
+ (column) => column.id === "environment",
+ ),
+ ).toMatchObject({ options: [{ value: "production" }] });
+ expect(
+ result.current.builderColumns.find((column) => column.id === "tags"),
+ ).toMatchObject({ options: [{ value: "customer-facing" }] });
+ expect(
+ result.current.builderColumns.find(
+ (column) => column.id === "experimentDatasetId",
+ ),
+ ).toMatchObject({
+ options: [{ value: "dataset-id", displayValue: "Support tickets" }],
+ });
+ });
+});
diff --git a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/hooks/useSampleObservationFilterOptions.ts b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/hooks/useSampleObservationFilterOptions.ts
new file mode 100644
index 0000000000..0b0880c753
--- /dev/null
+++ b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/hooks/useSampleObservationFilterOptions.ts
@@ -0,0 +1,150 @@
+import { useMemo } from "react";
+import {
+ eventsEvalFilterColumns,
+ experimentEvalFilterColsWithOptions,
+ observationEvalFilterColsWithOptions,
+ type ColumnDefinition,
+ type FilterState,
+ type TimeFilter,
+} from "@langfuse/shared";
+
+import { useEventsFilterOptions } from "@/src/features/events/hooks/useEventsFilterOptions";
+import {
+ type FieldRegistry,
+ withFieldOptions,
+} from "@/src/features/search-bar/lib/fields";
+import {
+ toObservedOptions,
+ type ObservedOptions,
+} from "@/src/features/search-bar/lib/observed-options";
+import {
+ removeInternalEvaluationEnvironmentColumnOptions,
+ removeInternalEvaluationEnvironmentOptions,
+} from "@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/buildSampleQueryFilters";
+import {
+ DATASET_NAME_COLUMN,
+ DATASET_NAME_FILTER_COLUMN,
+ addDatasetNameObservedOptions,
+ type DatasetFilterOption,
+} from "@/src/features/evals/v2/utils/datasetNameFilter";
+import type { EvaluatorFilterExperience } from "@/src/features/evals/v2/types/evaluatorFilterExperience";
+
+export type MapSampleObservedOptions = (
+ observed: ObservedOptions | undefined,
+) => ObservedOptions | undefined;
+
+const BUILDER_FILTER_OPTION_COLUMNS = [
+ "environment",
+ "name",
+ "traceTags",
+ "traceName",
+ "calledToolNames",
+ "experimentDatasetId",
+] satisfies NonNullable<
+ Parameters<typeof useEventsFilterOptions>[0]["columns"]
+>;
+
+type SampleObservationFilterOptionsProps = {
+ projectId: string;
+ startTimeFilter: TimeFilter[];
+ refiningFilter: FilterState;
+ filterMode: EvaluatorFilterExperience;
+ datasetOptions: DatasetFilterOption[];
+ mapObservedOptions: MapSampleObservedOptions;
+ activeRegistry: FieldRegistry;
+};
+
+export function useSampleObservationFilterOptions({
+ projectId,
+ startTimeFilter,
+ refiningFilter,
+ filterMode,
+ datasetOptions,
+ mapObservedOptions,
+ activeRegistry,
+}: SampleObservationFilterOptionsProps) {
+ const options = useEventsFilterOptions({
+ projectId,
+ startTimeFilter,
+ refiningFilter,
+ includeApproxCount: true,
+ lazy: filterMode === "query",
+ columns:
+ filterMode === "builder" ? BUILDER_FILTER_OPTION_COLUMNS : undefined,
+ });
+ const { filterOptions, isFilterOptionsPending } = options;
+ const searchRegistry = useMemo(
+ () =>
+ withFieldOptions(
+ activeRegistry,
+ DATASET_NAME_COLUMN,
+ datasetOptions.map((dataset) => ({
+ value: dataset.id,
+ displayValue: dataset.name,
+ })),
+ ),
+ [activeRegistry, datasetOptions],
+ );
+
+ const observed = useMemo(() => {
+ const visibleOptions = removeInternalEvaluationEnvironmentOptions(
+ toObservedOptions(filterOptions, isFilterOptionsPending),
+ );
+ const mapped = mapObservedOptions(visibleOptions);
+ return addDatasetNameObservedOptions(mapped, datasetOptions);
+ }, [
+ datasetOptions,
+ filterOptions,
+ isFilterOptionsPending,
+ mapObservedOptions,
+ ]);
+
+ const builderColumns = useMemo<ColumnDefinition[]>(() => {
+ const supportsDatasetName = searchRegistry.fields.some(
+ (field) => field.id === DATASET_NAME_COLUMN,
+ );
+ const columnsWithOptions = removeInternalEvaluationEnvironmentColumnOptions(
+ experimentEvalFilterColsWithOptions(
+ filterOptions,
+ observationEvalFilterColsWithOptions(
+ { ...filterOptions, tags: filterOptions.traceTags },
+ [...eventsEvalFilterColumns],
+ ),
+ ),
+ ).filter(
+ (column) => !supportsDatasetName || column.id !== "experimentDatasetId",
+ );
+ const datasetColumn: ColumnDefinition = {
+ name: DATASET_NAME_FILTER_COLUMN.name,
+ id: "experimentDatasetId",
+ aliases: DATASET_NAME_FILTER_COLUMN.aliases,
+ type: "stringOptions",
+ internal: DATASET_NAME_FILTER_COLUMN.internal,
+ nullable: DATASET_NAME_FILTER_COLUMN.nullable,
+ options: datasetOptions.map((dataset) => ({
+ value: dataset.id,
+ displayValue: dataset.name,
+ })),
+ };
+
+ return supportsDatasetName
+ ? [...columnsWithOptions, datasetColumn]
+ : columnsWithOptions;
+ }, [datasetOptions, filterOptions, searchRegistry]);
+
+ const queryOnlyColumnIds = useMemo(
+ () =>
+ searchRegistry.fields
+ .filter((field) => field.directFilter === false)
+ .map((field) => field.filterColumn ?? field.id),
+ [searchRegistry],
+ );
+
+ return {
+ ...options,
+ searchRegistry,
+ observed,
+ builderColumns,
+ queryOnlyColumnIds,
+ };
+}
diff --git a/web/src/features/evals/v2/constants/evaluatorFilterExperience.ts b/web/src/features/evals/v2/constants/evaluatorFilterExperience.ts
new file mode 100644
index 0000000000..b9d30eda23
--- /dev/null
+++ b/web/src/features/evals/v2/constants/evaluatorFilterExperience.ts
@@ -0,0 +1,2 @@
+export const EVALUATOR_FILTER_EXPERIENCE_STORAGE_KEY =
+ "langfuse-evaluation-observation-filter-mode";
diff --git a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
index 1f66f216c3..380ef38909 100644
--- a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
+++ b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
@@ -16,6 +16,7 @@ import { usePeekNavigation } from "@/src/components/table/peek/hooks/usePeekNavi
import { TablePeekViewTraceDetail } from "@/src/components/table/peek/peek-trace-detail";
import { Button } from "@/src/components/ui/button";
import { ConfirmDialog } from "@/src/components/ui/confirm-dialog";
+import useLocalStorage from "@/src/components/useLocalStorage";
import { ResizableSplitLayout } from "@/src/components/ui/resizable-split-layout";
import { EvaluatorVersionHistorySheet } from "../components/Evaluators/EvaluatorVersionHistorySheet/EvaluatorVersionHistorySheet";
import type { EvaluatorVersion } from "../components/Evaluators/EvaluatorVersionHistorySheet/types";
@@ -45,6 +46,8 @@ import { DefaultModelChangeConfirmationDialog } from "@/src/features/evals/v2/co
import { useProjectDefaultModel } from "@/src/features/evals/v2/hooks/useProjectDefaultModel";
import { safeRandomUUID } from "@/src/utils/safe-random-uuid";
import { EvaluatorSavedDialogContainer } from "@/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer";
+import { EVALUATOR_FILTER_EXPERIENCE_STORAGE_KEY } from "@/src/features/evals/v2/constants/evaluatorFilterExperience";
+import type { EvaluatorFilterExperience } from "@/src/features/evals/v2/types/evaluatorFilterExperience";
import { useLangfuseCloudRegion } from "@/src/features/organizations/hooks";
import { useProject } from "@/src/features/projects/hooks";
import { EvaluatorBlockedBanner } from "@/src/features/evals/v2/components/Evaluators/EvaluatorBlockedBanner/EvaluatorBlockedBanner";
@@ -164,6 +167,10 @@ export function EvaluatorSetupPage(
const router = useRouter();
const utils = api.useUtils();
const capture = usePostHogClientCapture();
+ const [filterExperience] = useLocalStorage<EvaluatorFilterExperience>(
+ EVALUATOR_FILTER_EXPERIENCE_STORAGE_KEY,
+ "builder",
+ );
const canReactivate = useHasProjectAccess({
projectId,
scope: "evaluator:CUD",
@@ -496,6 +503,7 @@ export function EvaluatorSetupPage(
});
capture("evaluators:update", {
evaluatorType: state.type,
+ filterExperience,
...(definition.type === "LLM_AS_JUDGE"
? getJudgePromptAnalyticsProperties(definition.promptMessages)
: {}),
@@ -517,9 +525,8 @@ export function EvaluatorSetupPage(
description,
definition,
});
- capture(
- "evaluators:create",
- getEvaluatorCreationAnalyticsProperties({
+ capture("evaluators:create", {
+ ...getEvaluatorCreationAnalyticsProperties({
evaluatorType: state.type,
creationSource: props.creationSource,
sourceCodeLanguage:
@@ -543,7 +550,8 @@ export function EvaluatorSetupPage(
}
: undefined,
}),
- );
+ filterExperience,
+ });
initialSnapshot.current = getCurrentSnapshot(state);
await utils.evalsV2.filterOptions.invalidate({ projectId });
if (!shouldOfferRuleAttachment(evaluator)) {
diff --git a/web/src/features/evals/v2/types/evaluatorFilterExperience.ts b/web/src/features/evals/v2/types/evaluatorFilterExperience.ts
new file mode 100644
index 0000000000..80b44b48e0
--- /dev/null
+++ b/web/src/features/evals/v2/types/evaluatorFilterExperience.ts
@@ -0,0 +1 @@
+export type EvaluatorFilterExperience = "query" | "builder";
diff --git a/web/src/features/filters/components/filter-builder.tsx b/web/src/features/filters/components/filter-builder.tsx
index bf0d23a0c1..46ec2ff5d4 100644
--- a/web/src/features/filters/components/filter-builder.tsx
+++ b/web/src/features/filters/components/filter-builder.tsx
@@ -450,6 +450,7 @@ export function InlineFilterBuilder({
stringObjectValueOptions,
onStringObjectKeyChange,
compact = false,
+ subtleAddButton = false,
}: {
columns: ColumnDefinitionWithAlert[];
filterState: FilterState;
@@ -471,6 +472,8 @@ export function InlineFilterBuilder({
onStringObjectKeyChange?: (key: string) => void;
/** compact renders the width-constrained two-line layout instead of the legacy single-row table. */
compact?: boolean;
+ /** Renders the add action like the subtle prompt-editor "Add message" action. */
+ subtleAddButton?: boolean;
}) {
const [wipFilterState, _setWipFilterState] =
useState<WipFilterState>(filterState);
@@ -520,6 +523,7 @@ export function InlineFilterBuilder({
stringObjectValueOptions={stringObjectValueOptions}
onStringObjectKeyChange={onStringObjectKeyChange}
compact={compact}
+ subtleAddButton={subtleAddButton}
/>
</div>
);
@@ -566,6 +570,7 @@ function FilterBuilderForm({
stringObjectValueOptions = {},
onStringObjectKeyChange,
compact = false,
+ subtleAddButton = false,
aiFilter,
}: {
columnIdentifier: ColumnIdentifier;
@@ -585,6 +590,7 @@ function FilterBuilderForm({
onStringObjectKeyChange?: (key: string) => void;
/** compact renders each condition as a wrapping block for width-constrained inline builders instead of the wide table. */
compact?: boolean;
+ subtleAddButton?: boolean;
aiFilter?: AiFilterConfig;
}) {
const [showAiFilter, setShowAiFilter] = useState(false);
@@ -1128,10 +1134,12 @@ function FilterBuilderForm({
) : (
<tr key={i}>
<td className="p-1 text-sm">{connector}</td>
- <td className="flex gap-2 p-1">
- {/* selector of the column to be filtered */}
- {columnCombobox}
- {keyControl}
+ <td className="p-1">
+ <div className="flex gap-2">
+ {/* selector of the column to be filtered */}
+ {columnCombobox}
+ {keyControl}
+ </div>
</td>
<td className="p-1">{operatorSelect}</td>
<td className="p-1">{valueControl}</td>
@@ -1238,11 +1246,21 @@ function FilterBuilderForm({
<Button
onClick={() => addNewFilter()}
type="button" // required as it will otherwise submit forms where this component is used
- className={cn("mt-2", compact && "mt-4 self-start")}
- variant="outline"
+ className={cn(
+ subtleAddButton
+ ? "text-foreground hover:text-foreground mt-2 h-6 w-full justify-start gap-1.5 px-0 py-0 text-xs leading-none underline-offset-4 hover:bg-transparent hover:underline"
+ : "mt-2",
+ compact && !subtleAddButton && "mt-4 self-start",
+ )}
+ variant={subtleAddButton ? "ghost" : "outline"}
size="sm"
>
- <Plus className="mr-2 h-4 w-4" />
+ <Plus
+ className={cn(
+ "shrink-0",
+ subtleAddButton ? "h-3.5 w-3.5" : "mr-2 h-4 w-4",
+ )}
+ />
Add filter
</Button>
) : null}
diff --git a/web/src/features/filters/components/multi-select.clienttest.tsx b/web/src/features/filters/components/multi-select.clienttest.tsx
new file mode 100644
index 0000000000..3f6c8741b4
--- /dev/null
+++ b/web/src/features/filters/components/multi-select.clienttest.tsx
@@ -0,0 +1,59 @@
+import { fireEvent, render, screen, within } from "@testing-library/react";
+
+import { MultiSelect } from "./multi-select";
+
+vi.stubGlobal(
+ "ResizeObserver",
+ class ResizeObserver {
+ observe() {}
+ unobserve() {}
+ disconnect() {}
+ },
+);
+Element.prototype.scrollIntoView = vi.fn();
+
+describe("MultiSelect", () => {
+ it("shows a selected value that is missing from the current options", () => {
+ render(
+ <MultiSelect
+ values={["GENERATION"]}
+ options={[]}
+ onValueChange={vi.fn()}
+ />,
+ );
+
+ expect(screen.getByText("GENERATION")).toBeInTheDocument();
+ });
+
+ it("shows edits to a selected custom value immediately", () => {
+ render(
+ <MultiSelect
+ values={["draft-value"]}
+ options={[]}
+ onValueChange={vi.fn()}
+ isCustomSelectEnabled
+ />,
+ );
+
+ const trigger = screen.getByRole("button");
+ fireEvent.click(trigger);
+ fireEvent.change(screen.getByPlaceholderText("Enter custom value"), {
+ target: { value: "edited-value" },
+ });
+
+ expect(within(trigger).getByText("edited-value")).toBeInTheDocument();
+ expect(within(trigger).queryByText("draft-value")).not.toBeInTheDocument();
+ });
+
+ it("summarizes several selected values that are missing from the options", () => {
+ render(
+ <MultiSelect
+ values={["one", "two", "three", "four", "five", "six"]}
+ options={[]}
+ onValueChange={vi.fn()}
+ />,
+ );
+
+ expect(screen.getByText("6 selected")).toBeInTheDocument();
+ });
+});
diff --git a/web/src/features/filters/components/multi-select.tsx b/web/src/features/filters/components/multi-select.tsx
index a4be749197..27692be614 100644
--- a/web/src/features/filters/components/multi-select.tsx
+++ b/web/src/features/filters/components/multi-select.tsx
@@ -131,18 +131,15 @@ export function MultiSelect({
};
function getSelectedOptions() {
- const selectedOptions = options.filter(({ value }) =>
- selectedValues.has(value),
+ const selectedOptions = mergedOptions.filter(
+ ({ value }) =>
+ selectedValues.has(value) &&
+ (!freeTextInput || value !== freeTextInput),
);
+ const liveCustomOption: FilterOption[] =
+ freeTextInput && freeText ? [{ value: freeText }] : [];
- const hasCustomOption =
- !!freeText &&
- !!getFreeTextInput(isCustomSelectEnabled, values, optionValues);
- const customOption: FilterOption[] = hasCustomOption
- ? [{ value: freeText }]
- : [];
-
- return [...selectedOptions, ...customOption];
+ return [...selectedOptions, ...liveCustomOption];
}
const selectedBadges =
diff --git a/web/src/features/filters/index.ts b/web/src/features/filters/index.ts
new file mode 100644
index 0000000000..8fae36ed2a
--- /dev/null
+++ b/web/src/features/filters/index.ts
@@ -0,0 +1 @@
+export { InlineFilterBuilder } from "@/src/features/filters/components/filter-builder";