🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(evals): simplify default environment filters (#16898)
Tobias Wochinger485d9ece3f9e
diff --git a/web/src/features/evals/eval-config-target-options.clienttest.ts b/web/src/features/evals/eval-config-target-options.clienttest.tsindex d91ebdd3d4..d4b2cb3139 100644--- a/web/src/features/evals/eval-config-target-options.clienttest.ts+++ b/web/src/features/evals/eval-config-target-options.clienttest.ts@@ -1,9 +1,16 @@// @vitest-environment node-import { EvalTargetObject } from "@langfuse/shared";+import {+ EvalTargetObject,+ LangfuseInternalTraceEnvironment,+} from "@langfuse/shared";import { resolveCheckboxOperator } from "@/src/features/filters/hooks/useSidebarFilterState";import { evalConfigFilterColumns } from "@/src/server/api/definitions/evalConfigsTable";-import { DEFAULT_OBSERVATION_FILTER_WHEN_REMAPPING } from "@/src/features/evals/utils/evaluator-constants";+import {+ DEFAULT_OBSERVATION_FILTER,+ DEFAULT_OBSERVATION_FILTER_WHEN_REMAPPING,+ DEFAULT_TRACE_FILTER,+} from "@/src/features/evals/utils/evaluator-constants";describe("eval config target behavior", () => {it("should exclude all non-trace targets when selecting trace", () => {@@ -39,6 +46,33 @@ describe("eval config target behavior", () => {});});+ it("keeps internal environment exclusions in legacy evaluator defaults", () => {+ const environmentFilter = {+ column: "environment",+ operator: "none of",+ value: [+ LangfuseInternalTraceEnvironment.LLMJudge,+ LangfuseInternalTraceEnvironment.CodeEval,+ LangfuseInternalTraceEnvironment.NaturalLanguageFilter,+ "langfuse-prompt-experiment",+ "langfuse-evaluation",+ "sdk-experiment",+ ],+ type: "stringOptions",+ };++ expect(DEFAULT_TRACE_FILTER).toEqual([environmentFilter]);+ expect(DEFAULT_OBSERVATION_FILTER).toEqual([+ {+ column: "type",+ operator: "any of",+ value: ["GENERATION"],+ type: "stringOptions",+ },+ environmentFilter,+ ]);+ });+it("uses semantic roots when remapping trace evaluators", () => {expect(DEFAULT_OBSERVATION_FILTER_WHEN_REMAPPING).toEqual([{diff --git a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/SampleObservationSelectorBase.tsx b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/SampleObservationSelectorBase.tsxindex 2d4d1ddde1..685a2b15de 100644--- a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/SampleObservationSelectorBase.tsx+++ b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/SampleObservationSelectorBase.tsx@@ -33,6 +33,10 @@ import { api, sendAsPostOption, type RouterOutputs } from "@/src/utils/api";import type { AbsoluteTimeRange } from "@/src/utils/date-range-utils";import { SectionHeader } from "@/src/features/evals/v2/components/Evaluators/Testing/components/SectionHeader/SectionHeader";import { EXPERIMENTS_AND_EVALS_EXCLUSION_FILTERS } from "@/src/features/evals/v2/constants/experimentAndEvalFilters";+import {+ buildSampleQueryFilters,+ removeInternalEvaluationEnvironmentOptions,+} from "@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/buildSampleQueryFilters";import { dedupeObservationPages } from "@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/dedupeObservations";import { toggleExampleFilters } from "@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/toggleExampleFilters";import {@@ -240,20 +244,25 @@ export function SampleObservationSelectorBase([timeRange],);const effectiveFilters = useMemo(- () => [...previewFilters, ...timeFilters(timeRange)],+ () => buildSampleQueryFilters(previewFilters, timeFilters(timeRange)),[previewFilters, timeRange],);+ const refiningFilter = useMemo(+ () => buildSampleQueryFilters(previewFilters),+ [previewFilters],+ );const options = useEventsFilterOptions({projectId,startTimeFilter,- refiningFilter: previewFilters,+ refiningFilter,includeApproxCount: true,lazy: true,});const observed = useMemo(() => {- const mapped = mapObservedOptions(+ const visibleOptions = removeInternalEvaluationEnvironmentOptions(toObservedOptions(options.filterOptions, options.isFilterOptionsPending),);+ const mapped = mapObservedOptions(visibleOptions);return addDatasetNameObservedOptions(mapped, datasetOptions);}, [datasetOptions,diff --git a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/buildSampleQueryFilters.clienttest.ts b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/buildSampleQueryFilters.clienttest.tsnew file mode 100644index 0000000000..977ae05279--- /dev/null+++ b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/buildSampleQueryFilters.clienttest.ts@@ -0,0 +1,61 @@+import type { FilterState } from "@langfuse/shared";+import { describe, expect, it } from "vitest";++import {+ buildSampleQueryFilters,+ removeInternalEvaluationEnvironmentOptions,+} from "./buildSampleQueryFilters";++describe("buildSampleQueryFilters", () => {+ it("hides internal evaluation environments without changing visible filters", () => {+ const visibleFilters = [+ {+ column: "type",+ type: "stringOptions" as const,+ operator: "any of" as const,+ value: ["GENERATION"],+ },+ ] satisfies FilterState;++ const queryFilters = buildSampleQueryFilters(visibleFilters);++ expect(queryFilters).toEqual([+ ...visibleFilters,+ {+ column: "environment",+ type: "stringOptions",+ operator: "none of",+ value: expect.arrayContaining([+ "langfuse-llm-as-a-judge",+ "langfuse-code-eval",+ "langfuse-natural-language-filter",+ "langfuse-prompt-experiment",+ "langfuse-in-app-agent",+ "langfuse-evaluation",+ "llm-as-a-judge",+ ]),+ },+ ]);+ expect(visibleFilters).toHaveLength(1);+ });++ it("keeps valid experiment and user environment options visible", () => {+ expect(+ removeInternalEvaluationEnvironmentOptions({+ environment: [+ { value: "langfuse-code-eval" },+ { value: "llm-as-a-judge" },+ { value: "sdk-experiment" },+ { value: "my-langfuse-production" },+ { value: "production" },+ ],+ }),+ ).toEqual({+ environment: [+ { value: "sdk-experiment" },+ { value: "my-langfuse-production" },+ { value: "production" },+ ],+ });+ });+});diff --git a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/buildSampleQueryFilters.ts b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/buildSampleQueryFilters.tsnew file mode 100644index 0000000000..6286c082bc--- /dev/null+++ b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/buildSampleQueryFilters.ts@@ -0,0 +1,34 @@+import type { FilterState } from "@langfuse/shared";++import type { ObservedOptions } from "@/src/features/search-bar/lib/observed-options";+import {+ INTERNAL_EVALUATION_ENVIRONMENTS,+ INTERNAL_EVALUATION_ENVIRONMENT_FILTERS,+} from "@/src/features/evals/v2/constants/experimentAndEvalFilters";++export function buildSampleQueryFilters(+ visibleFilters: FilterState,+ additionalFilters: FilterState = [],+): FilterState {+ return [+ ...visibleFilters,+ ...INTERNAL_EVALUATION_ENVIRONMENT_FILTERS,+ ...additionalFilters,+ ];+}++export function removeInternalEvaluationEnvironmentOptions(+ observed: ObservedOptions | undefined,+): ObservedOptions | undefined {+ if (observed?.environment === undefined) return observed;++ const internalEnvironments = new Set<string>(+ INTERNAL_EVALUATION_ENVIRONMENTS,+ );+ return {+ ...observed,+ environment: observed.environment.filter(+ ({ value }) => !internalEnvironments.has(value),+ ),+ };+}diff --git a/web/src/features/evals/v2/components/Rules/CreateRuleDialog/components/CreateRuleDialogContent/CreateRuleDialogContent.tsx b/web/src/features/evals/v2/components/Rules/CreateRuleDialog/components/CreateRuleDialogContent/CreateRuleDialogContent.tsxindex 9d3cc81fc5..4b8eb67a68 100644--- a/web/src/features/evals/v2/components/Rules/CreateRuleDialog/components/CreateRuleDialogContent/CreateRuleDialogContent.tsx+++ b/web/src/features/evals/v2/components/Rules/CreateRuleDialog/components/CreateRuleDialogContent/CreateRuleDialogContent.tsx@@ -19,14 +19,10 @@ import { showSuccessToast } from "@/src/features/notifications/showSuccessToast"import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";import { api } from "@/src/utils/api";import { trpcErrorToast } from "@/src/utils/trpcErrorToast";-import { EXPERIMENTS_AND_EVALS_EXCLUSION_FILTERS } from "@/src/features/evals/v2/constants/experimentAndEvalFilters";import { useLangfuseCloudRegion } from "@/src/features/organizations/hooks";import { useProject } from "@/src/features/projects/hooks";import { prepareNameForSave } from "@/src/features/evals/v2/fns/prepareNameForSave";--function resolveInitialRuleFilters(initialFilter?: FilterState) {- return initialFilter ?? EXPERIMENTS_AND_EVALS_EXCLUSION_FILTERS;-}+import { resolveInitialRuleFilters } from "./resolveInitialRuleFilters";export function CreateRuleDialogContent({projectId,diff --git a/web/src/features/evals/v2/components/Rules/CreateRuleDialog/components/CreateRuleDialogContent/resolveInitialRuleFilters.clienttest.ts b/web/src/features/evals/v2/components/Rules/CreateRuleDialog/components/CreateRuleDialogContent/resolveInitialRuleFilters.clienttest.tsnew file mode 100644index 0000000000..86074a12bb--- /dev/null+++ b/web/src/features/evals/v2/components/Rules/CreateRuleDialog/components/CreateRuleDialogContent/resolveInitialRuleFilters.clienttest.ts@@ -0,0 +1,29 @@+import { describe, expect, it } from "vitest";++import { resolveInitialRuleFilters } from "./resolveInitialRuleFilters";++describe("resolveInitialRuleFilters", () => {+ it("starts new rules with a root span filter", () => {+ expect(resolveInitialRuleFilters()).toEqual([+ {+ column: "isRootObservation",+ type: "boolean",+ operator: "=",+ value: true,+ },+ ]);+ });++ it("preserves supplied filters", () => {+ const filters = [+ {+ column: "type",+ type: "stringOptions" as const,+ operator: "any of" as const,+ value: ["GENERATION"],+ },+ ];++ expect(resolveInitialRuleFilters(filters)).toBe(filters);+ });+});diff --git a/web/src/features/evals/v2/components/Rules/CreateRuleDialog/components/CreateRuleDialogContent/resolveInitialRuleFilters.ts b/web/src/features/evals/v2/components/Rules/CreateRuleDialog/components/CreateRuleDialogContent/resolveInitialRuleFilters.tsnew file mode 100644index 0000000000..a445e9550c--- /dev/null+++ b/web/src/features/evals/v2/components/Rules/CreateRuleDialog/components/CreateRuleDialogContent/resolveInitialRuleFilters.ts@@ -0,0 +1,16 @@+import type { FilterState } from "@langfuse/shared";++export function resolveInitialRuleFilters(+ initialFilter?: FilterState,+): FilterState {+ return (+ initialFilter ?? [+ {+ column: "isRootObservation",+ type: "boolean",+ operator: "=",+ value: true,+ },+ ]+ );+}diff --git a/web/src/features/evals/v2/constants/experimentAndEvalFilters.ts b/web/src/features/evals/v2/constants/experimentAndEvalFilters.tsindex eb3886986a..03e74106db 100644--- a/web/src/features/evals/v2/constants/experimentAndEvalFilters.ts+++ b/web/src/features/evals/v2/constants/experimentAndEvalFilters.ts@@ -1,12 +1,26 @@-import type { FilterState } from "@langfuse/shared";+import {+ LangfuseInternalTraceEnvironment,+ type FilterState,+} from "@langfuse/shared";-export const EXPERIMENTS_AND_EVALS_EXCLUSION_FILTERS = [+export const INTERNAL_EVALUATION_ENVIRONMENTS = [+ ...Object.values(LangfuseInternalTraceEnvironment),+ "langfuse-evaluation",+ "langfuse",+ "llm-as-a-judge",+] as const;++export const INTERNAL_EVALUATION_ENVIRONMENT_FILTERS = [{column: "environment",- type: "string",- operator: "does not contain",- value: "langfuse-",+ type: "stringOptions",+ operator: "none of",+ value: [...INTERNAL_EVALUATION_ENVIRONMENTS],},+] satisfies FilterState;++export const EXPERIMENTS_AND_EVALS_EXCLUSION_FILTERS = [+ ...INTERNAL_EVALUATION_ENVIRONMENT_FILTERS,{column: "environment",type: "stringOptions",diff --git a/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.clienttest.ts b/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.clienttest.tsindex e5b75f47d0..fd5d387976 100644--- a/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.clienttest.ts+++ b/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.clienttest.ts@@ -2,7 +2,6 @@ import type { FilterState } from "@langfuse/shared";import { describe, expect, it } from "vitest";import { getDefaultCodeEvalSource } from "@/src/features/evals/utils/code-eval-template-starter-examples";-import { EXPERIMENTS_AND_EVALS_EXCLUSION_FILTERS } from "@/src/features/evals/v2/constants/experimentAndEvalFilters";import {createEvaluatorSetupStore,selectHasValidModel,@@ -95,7 +94,6 @@ describe("createEvaluatorSetupStore", () => {operator: "=",value: true,},- ...EXPERIMENTS_AND_EVALS_EXCLUSION_FILTERS,],});});@@ -213,7 +211,6 @@ describe("createEvaluatorSetupStore", () => {operator: "=",value: true,},- ...EXPERIMENTS_AND_EVALS_EXCLUSION_FILTERS,]);});diff --git a/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.ts b/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.tsindex 97f0bd0c0d..e45a31da38 100644--- a/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.ts+++ b/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.ts@@ -21,7 +21,6 @@ import type { JudgeModel } from "@/src/features/evals/v2/judgeModel";import type { ScoreOutputFormState } from "@/src/features/evals/v2/scoreOutputTypes";import type { NormalizedEvaluatorDefinition } from "@/src/features/evals/v2/server/evaluators/evaluatorTypes";import { toScoreOutputFormState } from "@/src/features/evals/v2/fns/scoreOutput/toScoreOutputFormState";-import { EXPERIMENTS_AND_EVALS_EXCLUSION_FILTERS } from "@/src/features/evals/v2/constants/experimentAndEvalFilters";import { safeRandomUUID } from "@/src/utils/safe-random-uuid";const DEFAULT_PROMPT = `Evaluate the quality of the response.@@ -197,7 +196,6 @@ export function createEvaluatorSetupStore({selectedObservation: null,sampleFilter: initialSampleFilter ?? [...DEFAULT_OBSERVATION_FILTER_WHEN_REMAPPING,- ...EXPERIMENTS_AND_EVALS_EXCLUSION_FILTERS,],promptPreviewEnabled: false,testPanelOpen: true,