🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(evals): add observation filter builder (#16930)
Tobias Wochinger858377d0a084
diff --git a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/SampleObservationSelectorBase.tsx b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/SampleObservationSelectorBase.tsxindex e20bdd8101..ed7a6b48cc 100644--- a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/SampleObservationSelectorBase.tsx+++ b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/SampleObservationSelectorBase.tsx@@ -15,36 +15,33 @@ import { useRowHeightLocalStorage } from "@/src/components/table/data-table-row-import type { LangfuseColumnDef } from "@/src/components/table/types";import { Button } from "@/src/components/ui/button";import { Skeleton } from "@/src/components/ui/skeleton";-import { useEventsFilterOptions } from "@/src/features/events/hooks/useEventsFilterOptions";+import useLocalStorage from "@/src/components/useLocalStorage";import { EventsSearchBarRow } from "@/src/features/search-bar/components/EventsSearchBarRow";import { useEventsSearchBar } from "@/src/features/search-bar/hooks/useEventsSearchBar";import { buildAiContext } from "@/src/features/search-bar/lib/ai-context";import {type FieldRegistry,EVENTS_FIELD_REGISTRY,- withFieldOptions,} from "@/src/features/search-bar/lib/fields";-import {- observedScoreNamesFromOptions,- toObservedOptions,-} from "@/src/features/search-bar/lib/observed-options";+import { observedScoreNamesFromOptions } from "@/src/features/search-bar/lib/observed-options";import useColumnOrder from "@/src/features/column-visibility/hooks/useColumnOrder";import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";import { api, sendAsPostOption, type RouterOutputs } from "@/src/utils/api";import type { AbsoluteTimeRange } from "@/src/utils/date-range-utils";import { SectionHeader } from "@/src/features/evals/v2/components/Evaluators/Testing/components/SectionHeader/SectionHeader";+import { EVALUATOR_FILTER_EXPERIENCE_STORAGE_KEY } from "@/src/features/evals/v2/constants/evaluatorFilterExperience";import { EXPERIMENTS_AND_EVALS_EXCLUSION_FILTERS } from "@/src/features/evals/v2/constants/experimentAndEvalFilters";-import {- buildSampleQueryFilters,- removeInternalEvaluationEnvironmentOptions,-} from "@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/buildSampleQueryFilters";+import { FilterModeToggle } from "@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/components/FilterModeToggle";+import { ObservationFilterBuilder } from "@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/components/ObservationFilterBuilder/ObservationFilterBuilder";+import { buildSampleQueryFilters } from "@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/buildSampleQueryFilters";import { dedupeObservationPages } from "@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/dedupeObservations";import { toggleExampleFilters } from "@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/toggleExampleFilters";import {- DATASET_NAME_COLUMN,- addDatasetNameObservedOptions,-} from "@/src/features/evals/v2/utils/datasetNameFilter";+ type MapSampleObservedOptions,+ useSampleObservationFilterOptions,+} from "@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/hooks/useSampleObservationFilterOptions";import { useReusableRuleFilterPresets } from "@/src/features/evals/v2/hooks/useReusableRuleFilterPresets";+import type { EvaluatorFilterExperience } from "@/src/features/evals/v2/types/evaluatorFilterExperience";import { usePostHogClientCapture } from "@/src/features/posthog-analytics";export type SampleObservation =@@ -152,9 +149,7 @@ export type SampleObservationSelectorBaseProps = {matchingDescription: string;matchingTooltip: string;formatCount: (count: number) => string;- mapObservedOptions: (- observed: ReturnType<typeof toObservedOptions>,- ) => ReturnType<typeof toObservedOptions>;+ mapObservedOptions: MapSampleObservedOptions;};export function SampleObservationSelectorBase(@@ -182,6 +177,11 @@ export function SampleObservationSelectorBase(mapObservedOptions,} = props;const activeRegistry = registry ?? EVENTS_FIELD_REGISTRY;+ const [filterMode, setFilterMode] =+ useLocalStorage<EvaluatorFilterExperience>(+ EVALUATOR_FILTER_EXPERIENCE_STORAGE_KEY,+ "builder",+ );const datasets = api.datasets.allDatasetMeta.useQuery({ projectId },{@@ -192,39 +192,6 @@ export function SampleObservationSelectorBase(},);const datasetOptions = useMemo(() => datasets.data ?? [], [datasets.data]);- const searchRegistry = useMemo(- () =>- withFieldOptions(- activeRegistry,- DATASET_NAME_COLUMN,- datasetOptions.map((dataset) => ({- value: dataset.id,- displayValue: dataset.name,- })),- ),- [activeRegistry, datasetOptions],- );- const reusableRuleFilters = useReusableRuleFilterPresets(- projectId,- searchRegistry,- );- const capture = usePostHogClientCapture();- const onQueryPresetPick = useCallback(- (presetId: string) => {- const preset = reusableRuleFilters.presets.find(- (candidate) => candidate.id === presetId,- );- if (!preset) return;- capture("evaluation_rules:filter_reused", {- tableName,- evaluatorCount: preset.evaluatorCount,- filterCount: preset.filterCount,- replacedFilterCount: filterState.length,- isV4: true,- });- },- [capture, filterState.length, reusableRuleFilters.presets, tableName],- );const setFilters = (next: FilterState | ((current: FilterState) => FilterState),) => {@@ -252,25 +219,38 @@ export function SampleObservationSelectorBase(() => buildSampleQueryFilters(previewFilters),[previewFilters],);- const options = useEventsFilterOptions({+ const options = useSampleObservationFilterOptions({projectId,startTimeFilter,refiningFilter,- includeApproxCount: true,- lazy: true,- });- const observed = useMemo(() => {- const visibleOptions = removeInternalEvaluationEnvironmentOptions(- toObservedOptions(options.filterOptions, options.isFilterOptionsPending),- );- const mapped = mapObservedOptions(visibleOptions);- return addDatasetNameObservedOptions(mapped, datasetOptions);- }, [+ filterMode,+ activeRegistry,datasetOptions,mapObservedOptions,- options.filterOptions,- options.isFilterOptionsPending,- ]);+ });+ const { searchRegistry, observed, builderColumns, queryOnlyColumnIds } =+ options;+ const reusableRuleFilters = useReusableRuleFilterPresets(+ projectId,+ searchRegistry,+ );+ const capture = usePostHogClientCapture();+ const onQueryPresetPick = useCallback(+ (presetId: string) => {+ const preset = reusableRuleFilters.presets.find(+ (candidate) => candidate.id === presetId,+ );+ if (!preset) return;+ capture("evaluation_rules:filter_reused", {+ tableName,+ evaluatorCount: preset.evaluatorCount,+ filterCount: preset.filterCount,+ replacedFilterCount: filterState.length,+ isV4: true,+ });+ },+ [capture, filterState.length, reusableRuleFilters.presets, tableName],+ );const search = useEventsSearchBar({projectId,tableName,@@ -498,11 +478,21 @@ export function SampleObservationSelectorBase(meta={null}description={filterDescription}tooltip={filterTooltip}- trailing={null}+ trailing={+ <FilterModeToggle+ mode={filterMode}+ onChange={(mode) => {+ if (mode === "builder" && searchQuery !== null) {+ search.applyFilters(filterState);+ }+ setFilterMode(mode);+ }}+ />+ }/>{datasets.isPending ? (<Skeleton className="h-10 w-full" />- ) : (+ ) : filterMode === "query" ? (<EventsSearchBarRowprojectId={projectId}tableName={tableName}@@ -519,26 +509,35 @@ export function SampleObservationSelectorBase(aiScoreNames={aiScoreNames}className="p-0"/>+ ) : (+ <ObservationFilterBuilder+ columns={builderColumns}+ filterState={filterState}+ onChange={setFilters}+ queryOnlyColumnIds={queryOnlyColumnIds}+ />)}- <div className="flex flex-wrap gap-2">- {EXAMPLES.map((example) => (- <Button- key={example.label}- type="button"- variant="outline"- size="sm"- className="flex h-8 items-center gap-2 text-sm"- onClick={() => {- setFilters((current) =>- toggleExampleFilters(current, [...example.filters]),- );- }}- >- <example.icon className="h-4 w-4" />- <span>{example.label}</span>- </Button>- ))}- </div>+ {filterMode === "query" ? (+ <div className="flex flex-wrap gap-2">+ {EXAMPLES.map((example) => (+ <Button+ key={example.label}+ type="button"+ variant="outline"+ size="sm"+ className="flex h-8 items-center gap-2 text-sm"+ onClick={() => {+ setFilters((current) =>+ toggleExampleFilters(current, [...example.filters]),+ );+ }}+ >+ <example.icon className="h-4 w-4" />+ <span>{example.label}</span>+ </Button>+ ))}+ </div>+ ) : null}</section><section className="flex min-h-0 flex-col gap-2">diff --git a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/components/FilterModeToggle.tsx b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/components/FilterModeToggle.tsxnew file mode 100644index 0000000000..36b363a4f5--- /dev/null+++ b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/components/FilterModeToggle.tsx@@ -0,0 +1,40 @@+import { Braces, ListFilter } from "lucide-react";++import type { EvaluatorFilterExperience } from "@/src/features/evals/v2/types/evaluatorFilterExperience";+import { cn } from "@/src/utils/tailwind";++export function FilterModeToggle({+ mode,+ onChange,+}: {+ mode: EvaluatorFilterExperience;+ onChange: (mode: EvaluatorFilterExperience) => void;+}) {+ return (+ <div+ className="bg-muted flex items-center rounded-md p-0.5"+ aria-label="Filter editor mode"+ >+ {(+ [+ { id: "query", label: "Query", icon: Braces },+ { id: "builder", label: "Builder", icon: ListFilter },+ ] as const+ ).map((option) => (+ <button+ key={option.id}+ type="button"+ aria-pressed={mode === option.id}+ onClick={() => onChange(option.id)}+ className={cn(+ "text-muted-foreground flex h-7 items-center gap-1 rounded px-2 text-xs",+ mode === option.id && "bg-background text-foreground shadow-sm",+ )}+ >+ <option.icon className="h-3.5 w-3.5" aria-hidden="true" />+ {option.label}+ </button>+ ))}+ </div>+ );+}diff --git a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/components/ObservationFilterBuilder/ObservationFilterBuilder.clienttest.tsx b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/components/ObservationFilterBuilder/ObservationFilterBuilder.clienttest.tsxnew file mode 100644index 0000000000..0facfca97e--- /dev/null+++ b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/components/ObservationFilterBuilder/ObservationFilterBuilder.clienttest.tsx@@ -0,0 +1,62 @@+import { render, screen } from "@testing-library/react";+import type { ColumnDefinition, FilterState } from "@langfuse/shared";++import { ObservationFilterBuilder } from "./ObservationFilterBuilder";++let containerWidth = 560;++vi.mock("@/src/hooks/useElementSize", () => ({+ useElementSize: () => [+ { current: null },+ { width: containerWidth, height: 200 },+ ],+}));++const columns: ColumnDefinition[] = [+ {+ id: "environment",+ name: "Environment",+ type: "stringOptions",+ internal: "environment",+ options: [{ value: "production" }],+ },+];++const filterState: FilterState = [+ {+ column: "environment",+ type: "stringOptions",+ operator: "any of",+ value: ["production"],+ },+];++const renderBuilder = () =>+ render(+ <ObservationFilterBuilder+ columns={columns}+ filterState={filterState}+ onChange={vi.fn()}+ queryOnlyColumnIds={[]}+ />,+ );++describe("ObservationFilterBuilder", () => {+ beforeEach(() => {+ containerWidth = 560;+ });++ it("keeps the table layout when space is limited", () => {+ renderBuilder();++ expect(screen.getByRole("table")).toBeInTheDocument();+ });++ it("wraps into the compact row layout when space is too narrow", () => {+ containerWidth = 420;+ renderBuilder();++ expect(screen.queryByRole("table")).not.toBeInTheDocument();+ expect(screen.getByText("Where")).toBeInTheDocument();+ });+});diff --git a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/components/ObservationFilterBuilder/ObservationFilterBuilder.stories.tsx b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/components/ObservationFilterBuilder/ObservationFilterBuilder.stories.tsxnew file mode 100644index 0000000000..2478a28220--- /dev/null+++ b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/components/ObservationFilterBuilder/ObservationFilterBuilder.stories.tsx@@ -0,0 +1,82 @@+import {+ eventsEvalFilterColumns,+ type ColumnDefinition,+ type FilterState,+} from "@langfuse/shared";+import { fn } from "storybook/test";++import preview from "../../../../../../../../../../../.storybook/preview";+import { ObservationFilterBuilder } from "./ObservationFilterBuilder";++const columns: ColumnDefinition[] = eventsEvalFilterColumns.map((column) => {+ if (column.id !== "environment" || column.type !== "stringOptions") {+ return column;+ }+ return {+ ...column,+ options: [{ value: "production" }, { value: "sdk-experiment" }],+ };+});++const filterState = [+ {+ column: "isRootObservation",+ type: "boolean",+ operator: "=",+ value: true,+ },+ {+ column: "environment",+ type: "stringOptions",+ operator: "any of",+ value: ["production"],+ },+ {+ column: "type",+ type: "stringOptions",+ operator: "any of",+ value: ["GENERATION"],+ },+ {+ column: "experimentId",+ type: "null",+ operator: "is null",+ value: "",+ },+] satisfies FilterState;++const meta = preview.meta({ component: ObservationFilterBuilder });++const args = {+ columns,+ filterState,+ onChange: fn(),+ queryOnlyColumnIds: ["metadata"],+};++export const Wide = meta.story({+ args,+ render: (storyArgs) => (+ <div className="max-w-5xl">+ <ObservationFilterBuilder {...storyArgs} />+ </div>+ ),+});++export const Narrow = meta.story({+ args,+ render: (storyArgs) => (+ <div className="w-[560px] max-w-full">+ <ObservationFilterBuilder {...storyArgs} />+ </div>+ ),+});++export const Compact = meta.story({+ args,+ render: (storyArgs) => (+ <div className="w-[420px] max-w-full">+ <ObservationFilterBuilder {...storyArgs} />+ </div>+ ),+});diff --git a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/components/ObservationFilterBuilder/ObservationFilterBuilder.tsx b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/components/ObservationFilterBuilder/ObservationFilterBuilder.tsxnew file mode 100644index 0000000000..4e08e7d43a--- /dev/null+++ b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/components/ObservationFilterBuilder/ObservationFilterBuilder.tsx@@ -0,0 +1,50 @@+import type { ColumnDefinition, FilterState } from "@langfuse/shared";++import { InlineFilterBuilder } from "@/src/features/filters";+import { useElementSize } from "@/src/hooks/useElementSize";+import { cn } from "@/src/utils/tailwind";++export function ObservationFilterBuilder({+ columns,+ filterState,+ onChange,+ queryOnlyColumnIds,+}: {+ columns: ColumnDefinition[];+ filterState: FilterState;+ onChange: (filters: FilterState) => void;+ queryOnlyColumnIds: string[];+}) {+ const [containerRef, containerSize] = useElementSize<HTMLDivElement>();+ const containerWidth = containerSize?.width;+ const isNarrow = containerWidth !== undefined && containerWidth < 720;+ const isCompact = containerWidth !== undefined && containerWidth < 480;++ return (+ <div+ ref={containerRef}+ className={cn(+ "[&_table]:w-full [&_table]:table-fixed [&_td:first-child]:w-16 [&_td:last-child]:w-8 [&_td:last-child]:text-right",+ isNarrow+ ? "[&_td:nth-child(2)]:w-40 [&_td:nth-child(3)]:w-32"+ : "[&_td:nth-child(2)]:w-56 [&_td:nth-child(3)]:w-40",+ )}+ >+ <InlineFilterBuilder+ columns={columns}+ filterState={filterState}+ onChange={onChange}+ columnIdentifier="id"+ columnsHiddenUnlessSelected={queryOnlyColumnIds}+ compact={isCompact}+ subtleAddButton+ columnsWithCustomSelect={[+ "tags",+ "name",+ "traceName",+ "calledToolNames",+ ]}+ />+ </div>+ );+}diff --git a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/buildSampleQueryFilters.clienttest.ts b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/buildSampleQueryFilters.clienttest.tsindex 977ae05279..1e856c6947 100644--- a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/buildSampleQueryFilters.clienttest.ts+++ b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/buildSampleQueryFilters.clienttest.ts@@ -1,8 +1,9 @@-import type { FilterState } from "@langfuse/shared";+import type { ColumnDefinition, FilterState } from "@langfuse/shared";import { describe, expect, it } from "vitest";import {buildSampleQueryFilters,+ removeInternalEvaluationEnvironmentColumnOptions,removeInternalEvaluationEnvironmentOptions,} from "./buildSampleQueryFilters";@@ -39,6 +40,25 @@ describe("buildSampleQueryFilters", () => {expect(visibleFilters).toHaveLength(1);});+ it("hides internal evaluation environments from builder columns", () => {+ const columns: ColumnDefinition[] = [+ {+ id: "environment",+ name: "Environment",+ type: "stringOptions",+ internal: "environment",+ options: [{ value: "langfuse-code-eval" }, { value: "production" }],+ },+ ];++ expect(removeInternalEvaluationEnvironmentColumnOptions(columns)).toEqual([+ {+ ...columns[0],+ options: [{ value: "production" }],+ },+ ]);+ });+it("keeps valid experiment and user environment options visible", () => {expect(removeInternalEvaluationEnvironmentOptions({diff --git a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/buildSampleQueryFilters.ts b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/buildSampleQueryFilters.tsindex 6286c082bc..49f93ccbf5 100644--- a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/buildSampleQueryFilters.ts+++ b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/buildSampleQueryFilters.ts@@ -1,4 +1,4 @@-import type { FilterState } from "@langfuse/shared";+import type { ColumnDefinition, FilterState } from "@langfuse/shared";import type { ObservedOptions } from "@/src/features/search-bar/lib/observed-options";import {@@ -17,6 +17,25 @@ export function buildSampleQueryFilters(];}+export function removeInternalEvaluationEnvironmentColumnOptions(+ columns: ColumnDefinition[],+): ColumnDefinition[] {+ const internalEnvironments = new Set<string>(+ INTERNAL_EVALUATION_ENVIRONMENTS,+ );+ return columns.map((column) => {+ if (column.id !== "environment" || column.type !== "stringOptions") {+ return column;+ }+ return {+ ...column,+ options: column.options.filter(+ ({ value }) => !internalEnvironments.has(value),+ ),+ };+ });+}+export function removeInternalEvaluationEnvironmentOptions(observed: ObservedOptions | undefined,): ObservedOptions | undefined {diff --git a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/hooks/useSampleObservationFilterOptions.clienttest.ts b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/hooks/useSampleObservationFilterOptions.clienttest.tsnew file mode 100644index 0000000000..2d137ff309--- /dev/null+++ b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/hooks/useSampleObservationFilterOptions.clienttest.ts@@ -0,0 +1,82 @@+import { renderHook } from "@testing-library/react";++import { EVALUATOR_FIELD_REGISTRY } from "@/src/features/evals/v2/constants/evaluatorSearchRegistry";+import { useSampleObservationFilterOptions } from "./useSampleObservationFilterOptions";++const { useEventsFilterOptionsMock } = vi.hoisted(() => ({+ useEventsFilterOptionsMock: vi.fn(),+}));++vi.mock("@/src/features/events/hooks/useEventsFilterOptions", () => ({+ useEventsFilterOptions: useEventsFilterOptionsMock,+}));++const refiningFilter = [+ {+ column: "environment",+ type: "stringOptions" as const,+ operator: "any of" as const,+ value: ["production"],+ },+];++describe("useSampleObservationFilterOptions", () => {+ beforeEach(() => {+ useEventsFilterOptionsMock.mockReturnValue({+ filterOptions: {+ environment: [{ value: "langfuse-code-eval" }, { value: "production" }],+ traceTags: [{ value: "customer-facing" }],+ },+ isFilterOptionsPending: false,+ });+ });++ it("derives query and builder values from the same filter options", () => {+ const { result } = renderHook(() =>+ useSampleObservationFilterOptions({+ projectId: "project-id",+ startTimeFilter: [],+ refiningFilter,+ filterMode: "builder",+ datasetOptions: [{ id: "dataset-id", name: "Support tickets" }],+ mapObservedOptions: (observed) => observed,+ activeRegistry: EVALUATOR_FIELD_REGISTRY,+ }),+ );++ expect(useEventsFilterOptionsMock).toHaveBeenCalledWith(+ expect.objectContaining({+ refiningFilter,+ lazy: false,+ columns: expect.arrayContaining(["environment", "traceTags"]),+ }),+ );+ expect(+ result.current.searchRegistry.fields+ .find((field) => field.id === "datasetName")+ ?.displayValueByFilterValue?.get("dataset-id"),+ ).toBe("Support tickets");+ expect(result.current.observed?.environment).toEqual([+ { value: "production" },+ ]);+ expect(result.current.observed?.datasetName).toEqual([+ { value: "Support tickets" },+ ]);++ expect(+ result.current.builderColumns.find(+ (column) => column.id === "environment",+ ),+ ).toMatchObject({ options: [{ value: "production" }] });+ expect(+ result.current.builderColumns.find((column) => column.id === "tags"),+ ).toMatchObject({ options: [{ value: "customer-facing" }] });+ expect(+ result.current.builderColumns.find(+ (column) => column.id === "experimentDatasetId",+ ),+ ).toMatchObject({+ options: [{ value: "dataset-id", displayValue: "Support tickets" }],+ });+ });+});diff --git a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/hooks/useSampleObservationFilterOptions.ts b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/hooks/useSampleObservationFilterOptions.tsnew file mode 100644index 0000000000..0b0880c753--- /dev/null+++ b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/hooks/useSampleObservationFilterOptions.ts@@ -0,0 +1,150 @@+import { useMemo } from "react";+import {+ eventsEvalFilterColumns,+ experimentEvalFilterColsWithOptions,+ observationEvalFilterColsWithOptions,+ type ColumnDefinition,+ type FilterState,+ type TimeFilter,+} from "@langfuse/shared";++import { useEventsFilterOptions } from "@/src/features/events/hooks/useEventsFilterOptions";+import {+ type FieldRegistry,+ withFieldOptions,+} from "@/src/features/search-bar/lib/fields";+import {+ toObservedOptions,+ type ObservedOptions,+} from "@/src/features/search-bar/lib/observed-options";+import {+ removeInternalEvaluationEnvironmentColumnOptions,+ removeInternalEvaluationEnvironmentOptions,+} from "@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/buildSampleQueryFilters";+import {+ DATASET_NAME_COLUMN,+ DATASET_NAME_FILTER_COLUMN,+ addDatasetNameObservedOptions,+ type DatasetFilterOption,+} from "@/src/features/evals/v2/utils/datasetNameFilter";+import type { EvaluatorFilterExperience } from "@/src/features/evals/v2/types/evaluatorFilterExperience";++export type MapSampleObservedOptions = (+ observed: ObservedOptions | undefined,+) => ObservedOptions | undefined;++const BUILDER_FILTER_OPTION_COLUMNS = [+ "environment",+ "name",+ "traceTags",+ "traceName",+ "calledToolNames",+ "experimentDatasetId",+] satisfies NonNullable<+ Parameters<typeof useEventsFilterOptions>[0]["columns"]+>;++type SampleObservationFilterOptionsProps = {+ projectId: string;+ startTimeFilter: TimeFilter[];+ refiningFilter: FilterState;+ filterMode: EvaluatorFilterExperience;+ datasetOptions: DatasetFilterOption[];+ mapObservedOptions: MapSampleObservedOptions;+ activeRegistry: FieldRegistry;+};++export function useSampleObservationFilterOptions({+ projectId,+ startTimeFilter,+ refiningFilter,+ filterMode,+ datasetOptions,+ mapObservedOptions,+ activeRegistry,+}: SampleObservationFilterOptionsProps) {+ const options = useEventsFilterOptions({+ projectId,+ startTimeFilter,+ refiningFilter,+ includeApproxCount: true,+ lazy: filterMode === "query",+ columns:+ filterMode === "builder" ? BUILDER_FILTER_OPTION_COLUMNS : undefined,+ });+ const { filterOptions, isFilterOptionsPending } = options;+ const searchRegistry = useMemo(+ () =>+ withFieldOptions(+ activeRegistry,+ DATASET_NAME_COLUMN,+ datasetOptions.map((dataset) => ({+ value: dataset.id,+ displayValue: dataset.name,+ })),+ ),+ [activeRegistry, datasetOptions],+ );++ const observed = useMemo(() => {+ const visibleOptions = removeInternalEvaluationEnvironmentOptions(+ toObservedOptions(filterOptions, isFilterOptionsPending),+ );+ const mapped = mapObservedOptions(visibleOptions);+ return addDatasetNameObservedOptions(mapped, datasetOptions);+ }, [+ datasetOptions,+ filterOptions,+ isFilterOptionsPending,+ mapObservedOptions,+ ]);++ const builderColumns = useMemo<ColumnDefinition[]>(() => {+ const supportsDatasetName = searchRegistry.fields.some(+ (field) => field.id === DATASET_NAME_COLUMN,+ );+ const columnsWithOptions = removeInternalEvaluationEnvironmentColumnOptions(+ experimentEvalFilterColsWithOptions(+ filterOptions,+ observationEvalFilterColsWithOptions(+ { ...filterOptions, tags: filterOptions.traceTags },+ [...eventsEvalFilterColumns],+ ),+ ),+ ).filter(+ (column) => !supportsDatasetName || column.id !== "experimentDatasetId",+ );+ const datasetColumn: ColumnDefinition = {+ name: DATASET_NAME_FILTER_COLUMN.name,+ id: "experimentDatasetId",+ aliases: DATASET_NAME_FILTER_COLUMN.aliases,+ type: "stringOptions",+ internal: DATASET_NAME_FILTER_COLUMN.internal,+ nullable: DATASET_NAME_FILTER_COLUMN.nullable,+ options: datasetOptions.map((dataset) => ({+ value: dataset.id,+ displayValue: dataset.name,+ })),+ };++ return supportsDatasetName+ ? [...columnsWithOptions, datasetColumn]+ : columnsWithOptions;+ }, [datasetOptions, filterOptions, searchRegistry]);++ const queryOnlyColumnIds = useMemo(+ () =>+ searchRegistry.fields+ .filter((field) => field.directFilter === false)+ .map((field) => field.filterColumn ?? field.id),+ [searchRegistry],+ );++ return {+ ...options,+ searchRegistry,+ observed,+ builderColumns,+ queryOnlyColumnIds,+ };+}diff --git a/web/src/features/evals/v2/constants/evaluatorFilterExperience.ts b/web/src/features/evals/v2/constants/evaluatorFilterExperience.tsnew file mode 100644index 0000000000..b9d30eda23--- /dev/null+++ b/web/src/features/evals/v2/constants/evaluatorFilterExperience.ts@@ -0,0 +1,2 @@+export const EVALUATOR_FILTER_EXPERIENCE_STORAGE_KEY =+ "langfuse-evaluation-observation-filter-mode";diff --git a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsxindex 1f66f216c3..380ef38909 100644--- a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx+++ b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx@@ -16,6 +16,7 @@ import { usePeekNavigation } from "@/src/components/table/peek/hooks/usePeekNaviimport { TablePeekViewTraceDetail } from "@/src/components/table/peek/peek-trace-detail";import { Button } from "@/src/components/ui/button";import { ConfirmDialog } from "@/src/components/ui/confirm-dialog";+import useLocalStorage from "@/src/components/useLocalStorage";import { ResizableSplitLayout } from "@/src/components/ui/resizable-split-layout";import { EvaluatorVersionHistorySheet } from "../components/Evaluators/EvaluatorVersionHistorySheet/EvaluatorVersionHistorySheet";import type { EvaluatorVersion } from "../components/Evaluators/EvaluatorVersionHistorySheet/types";@@ -45,6 +46,8 @@ import { DefaultModelChangeConfirmationDialog } from "@/src/features/evals/v2/coimport { useProjectDefaultModel } from "@/src/features/evals/v2/hooks/useProjectDefaultModel";import { safeRandomUUID } from "@/src/utils/safe-random-uuid";import { EvaluatorSavedDialogContainer } from "@/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer";+import { EVALUATOR_FILTER_EXPERIENCE_STORAGE_KEY } from "@/src/features/evals/v2/constants/evaluatorFilterExperience";+import type { EvaluatorFilterExperience } from "@/src/features/evals/v2/types/evaluatorFilterExperience";import { useLangfuseCloudRegion } from "@/src/features/organizations/hooks";import { useProject } from "@/src/features/projects/hooks";import { EvaluatorBlockedBanner } from "@/src/features/evals/v2/components/Evaluators/EvaluatorBlockedBanner/EvaluatorBlockedBanner";@@ -164,6 +167,10 @@ export function EvaluatorSetupPage(const router = useRouter();const utils = api.useUtils();const capture = usePostHogClientCapture();+ const [filterExperience] = useLocalStorage<EvaluatorFilterExperience>(+ EVALUATOR_FILTER_EXPERIENCE_STORAGE_KEY,+ "builder",+ );const canReactivate = useHasProjectAccess({projectId,scope: "evaluator:CUD",@@ -496,6 +503,7 @@ export function EvaluatorSetupPage(});capture("evaluators:update", {evaluatorType: state.type,+ filterExperience,...(definition.type === "LLM_AS_JUDGE"? getJudgePromptAnalyticsProperties(definition.promptMessages): {}),@@ -517,9 +525,8 @@ export function EvaluatorSetupPage(description,definition,});- capture(- "evaluators:create",- getEvaluatorCreationAnalyticsProperties({+ capture("evaluators:create", {+ ...getEvaluatorCreationAnalyticsProperties({evaluatorType: state.type,creationSource: props.creationSource,sourceCodeLanguage:@@ -543,7 +550,8 @@ export function EvaluatorSetupPage(}: undefined,}),- );+ filterExperience,+ });initialSnapshot.current = getCurrentSnapshot(state);await utils.evalsV2.filterOptions.invalidate({ projectId });if (!shouldOfferRuleAttachment(evaluator)) {diff --git a/web/src/features/evals/v2/types/evaluatorFilterExperience.ts b/web/src/features/evals/v2/types/evaluatorFilterExperience.tsnew file mode 100644index 0000000000..80b44b48e0--- /dev/null+++ b/web/src/features/evals/v2/types/evaluatorFilterExperience.ts@@ -0,0 +1 @@+export type EvaluatorFilterExperience = "query" | "builder";diff --git a/web/src/features/filters/components/filter-builder.tsx b/web/src/features/filters/components/filter-builder.tsxindex bf0d23a0c1..46ec2ff5d4 100644--- a/web/src/features/filters/components/filter-builder.tsx+++ b/web/src/features/filters/components/filter-builder.tsx@@ -450,6 +450,7 @@ export function InlineFilterBuilder({stringObjectValueOptions,onStringObjectKeyChange,compact = false,+ subtleAddButton = false,}: {columns: ColumnDefinitionWithAlert[];filterState: FilterState;@@ -471,6 +472,8 @@ export function InlineFilterBuilder({onStringObjectKeyChange?: (key: string) => void;/** compact renders the width-constrained two-line layout instead of the legacy single-row table. */compact?: boolean;+ /** Renders the add action like the subtle prompt-editor "Add message" action. */+ subtleAddButton?: boolean;}) {const [wipFilterState, _setWipFilterState] =useState<WipFilterState>(filterState);@@ -520,6 +523,7 @@ export function InlineFilterBuilder({stringObjectValueOptions={stringObjectValueOptions}onStringObjectKeyChange={onStringObjectKeyChange}compact={compact}+ subtleAddButton={subtleAddButton}/></div>);@@ -566,6 +570,7 @@ function FilterBuilderForm({stringObjectValueOptions = {},onStringObjectKeyChange,compact = false,+ subtleAddButton = false,aiFilter,}: {columnIdentifier: ColumnIdentifier;@@ -585,6 +590,7 @@ function FilterBuilderForm({onStringObjectKeyChange?: (key: string) => void;/** compact renders each condition as a wrapping block for width-constrained inline builders instead of the wide table. */compact?: boolean;+ subtleAddButton?: boolean;aiFilter?: AiFilterConfig;}) {const [showAiFilter, setShowAiFilter] = useState(false);@@ -1128,10 +1134,12 @@ function FilterBuilderForm({) : (<tr key={i}><td className="p-1 text-sm">{connector}</td>- <td className="flex gap-2 p-1">- {/* selector of the column to be filtered */}- {columnCombobox}- {keyControl}+ <td className="p-1">+ <div className="flex gap-2">+ {/* selector of the column to be filtered */}+ {columnCombobox}+ {keyControl}+ </div></td><td className="p-1">{operatorSelect}</td><td className="p-1">{valueControl}</td>@@ -1238,11 +1246,21 @@ function FilterBuilderForm({<ButtononClick={() => addNewFilter()}type="button" // required as it will otherwise submit forms where this component is used- className={cn("mt-2", compact && "mt-4 self-start")}- variant="outline"+ className={cn(+ subtleAddButton+ ? "text-foreground hover:text-foreground mt-2 h-6 w-full justify-start gap-1.5 px-0 py-0 text-xs leading-none underline-offset-4 hover:bg-transparent hover:underline"+ : "mt-2",+ compact && !subtleAddButton && "mt-4 self-start",+ )}+ variant={subtleAddButton ? "ghost" : "outline"}size="sm">- <Plus className="mr-2 h-4 w-4" />+ <Plus+ className={cn(+ "shrink-0",+ subtleAddButton ? "h-3.5 w-3.5" : "mr-2 h-4 w-4",+ )}+ />Add filter</Button>) : null}diff --git a/web/src/features/filters/components/multi-select.clienttest.tsx b/web/src/features/filters/components/multi-select.clienttest.tsxnew file mode 100644index 0000000000..3f6c8741b4--- /dev/null+++ b/web/src/features/filters/components/multi-select.clienttest.tsx@@ -0,0 +1,59 @@+import { fireEvent, render, screen, within } from "@testing-library/react";++import { MultiSelect } from "./multi-select";++vi.stubGlobal(+ "ResizeObserver",+ class ResizeObserver {+ observe() {}+ unobserve() {}+ disconnect() {}+ },+);+Element.prototype.scrollIntoView = vi.fn();++describe("MultiSelect", () => {+ it("shows a selected value that is missing from the current options", () => {+ render(+ <MultiSelect+ values={["GENERATION"]}+ options={[]}+ onValueChange={vi.fn()}+ />,+ );++ expect(screen.getByText("GENERATION")).toBeInTheDocument();+ });++ it("shows edits to a selected custom value immediately", () => {+ render(+ <MultiSelect+ values={["draft-value"]}+ options={[]}+ onValueChange={vi.fn()}+ isCustomSelectEnabled+ />,+ );++ const trigger = screen.getByRole("button");+ fireEvent.click(trigger);+ fireEvent.change(screen.getByPlaceholderText("Enter custom value"), {+ target: { value: "edited-value" },+ });++ expect(within(trigger).getByText("edited-value")).toBeInTheDocument();+ expect(within(trigger).queryByText("draft-value")).not.toBeInTheDocument();+ });++ it("summarizes several selected values that are missing from the options", () => {+ render(+ <MultiSelect+ values={["one", "two", "three", "four", "five", "six"]}+ options={[]}+ onValueChange={vi.fn()}+ />,+ );++ expect(screen.getByText("6 selected")).toBeInTheDocument();+ });+});diff --git a/web/src/features/filters/components/multi-select.tsx b/web/src/features/filters/components/multi-select.tsxindex a4be749197..27692be614 100644--- a/web/src/features/filters/components/multi-select.tsx+++ b/web/src/features/filters/components/multi-select.tsx@@ -131,18 +131,15 @@ export function MultiSelect({};function getSelectedOptions() {- const selectedOptions = options.filter(({ value }) =>- selectedValues.has(value),+ const selectedOptions = mergedOptions.filter(+ ({ value }) =>+ selectedValues.has(value) &&+ (!freeTextInput || value !== freeTextInput),);+ const liveCustomOption: FilterOption[] =+ freeTextInput && freeText ? [{ value: freeText }] : [];- const hasCustomOption =- !!freeText &&- !!getFreeTextInput(isCustomSelectEnabled, values, optionValues);- const customOption: FilterOption[] = hasCustomOption- ? [{ value: freeText }]- : [];-- return [...selectedOptions, ...customOption];+ return [...selectedOptions, ...liveCustomOption];}const selectedBadges =diff --git a/web/src/features/filters/index.ts b/web/src/features/filters/index.tsnew file mode 100644index 0000000000..8fae36ed2a--- /dev/null+++ b/web/src/features/filters/index.ts@@ -0,0 +1 @@+export { InlineFilterBuilder } from "@/src/features/filters/components/filter-builder";