Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(evals): simplify default environment filters (#16898)

Tobias Wochinger485d9ece3f9e
diff --git a/web/src/features/evals/eval-config-target-options.clienttest.ts b/web/src/features/evals/eval-config-target-options.clienttest.ts
index d91ebdd3d4..d4b2cb3139 100644
--- a/web/src/features/evals/eval-config-target-options.clienttest.ts
+++ b/web/src/features/evals/eval-config-target-options.clienttest.ts
@@ -1,9 +1,16 @@
// @vitest-environment node
-import { EvalTargetObject } from "@langfuse/shared";
+import {
+ EvalTargetObject,
+ LangfuseInternalTraceEnvironment,
+} from "@langfuse/shared";
import { resolveCheckboxOperator } from "@/src/features/filters/hooks/useSidebarFilterState";
import { evalConfigFilterColumns } from "@/src/server/api/definitions/evalConfigsTable";
-import { DEFAULT_OBSERVATION_FILTER_WHEN_REMAPPING } from "@/src/features/evals/utils/evaluator-constants";
+import {
+ DEFAULT_OBSERVATION_FILTER,
+ DEFAULT_OBSERVATION_FILTER_WHEN_REMAPPING,
+ DEFAULT_TRACE_FILTER,
+} from "@/src/features/evals/utils/evaluator-constants";
describe("eval config target behavior", () => {
it("should exclude all non-trace targets when selecting trace", () => {
@@ -39,6 +46,33 @@ describe("eval config target behavior", () => {
});
});
+ it("keeps internal environment exclusions in legacy evaluator defaults", () => {
+ const environmentFilter = {
+ column: "environment",
+ operator: "none of",
+ value: [
+ LangfuseInternalTraceEnvironment.LLMJudge,
+ LangfuseInternalTraceEnvironment.CodeEval,
+ LangfuseInternalTraceEnvironment.NaturalLanguageFilter,
+ "langfuse-prompt-experiment",
+ "langfuse-evaluation",
+ "sdk-experiment",
+ ],
+ type: "stringOptions",
+ };
+
+ expect(DEFAULT_TRACE_FILTER).toEqual([environmentFilter]);
+ expect(DEFAULT_OBSERVATION_FILTER).toEqual([
+ {
+ column: "type",
+ operator: "any of",
+ value: ["GENERATION"],
+ type: "stringOptions",
+ },
+ environmentFilter,
+ ]);
+ });
+
it("uses semantic roots when remapping trace evaluators", () => {
expect(DEFAULT_OBSERVATION_FILTER_WHEN_REMAPPING).toEqual([
{
diff --git a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/SampleObservationSelectorBase.tsx b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/SampleObservationSelectorBase.tsx
index 2d4d1ddde1..685a2b15de 100644
--- a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/SampleObservationSelectorBase.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/SampleObservationSelectorBase.tsx
@@ -33,6 +33,10 @@ import { api, sendAsPostOption, type RouterOutputs } from "@/src/utils/api";
import type { AbsoluteTimeRange } from "@/src/utils/date-range-utils";
import { SectionHeader } from "@/src/features/evals/v2/components/Evaluators/Testing/components/SectionHeader/SectionHeader";
import { EXPERIMENTS_AND_EVALS_EXCLUSION_FILTERS } from "@/src/features/evals/v2/constants/experimentAndEvalFilters";
+import {
+ buildSampleQueryFilters,
+ removeInternalEvaluationEnvironmentOptions,
+} from "@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/buildSampleQueryFilters";
import { dedupeObservationPages } from "@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/dedupeObservations";
import { toggleExampleFilters } from "@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/toggleExampleFilters";
import {
@@ -240,20 +244,25 @@ export function SampleObservationSelectorBase(
[timeRange],
);
const effectiveFilters = useMemo(
- () => [...previewFilters, ...timeFilters(timeRange)],
+ () => buildSampleQueryFilters(previewFilters, timeFilters(timeRange)),
[previewFilters, timeRange],
);
+ const refiningFilter = useMemo(
+ () => buildSampleQueryFilters(previewFilters),
+ [previewFilters],
+ );
const options = useEventsFilterOptions({
projectId,
startTimeFilter,
- refiningFilter: previewFilters,
+ refiningFilter,
includeApproxCount: true,
lazy: true,
});
const observed = useMemo(() => {
- const mapped = mapObservedOptions(
+ const visibleOptions = removeInternalEvaluationEnvironmentOptions(
toObservedOptions(options.filterOptions, options.isFilterOptionsPending),
);
+ const mapped = mapObservedOptions(visibleOptions);
return addDatasetNameObservedOptions(mapped, datasetOptions);
}, [
datasetOptions,
diff --git a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/buildSampleQueryFilters.clienttest.ts b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/buildSampleQueryFilters.clienttest.ts
new file mode 100644
index 0000000000..977ae05279
--- /dev/null
+++ b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/buildSampleQueryFilters.clienttest.ts
@@ -0,0 +1,61 @@
+import type { FilterState } from "@langfuse/shared";
+import { describe, expect, it } from "vitest";
+
+import {
+ buildSampleQueryFilters,
+ removeInternalEvaluationEnvironmentOptions,
+} from "./buildSampleQueryFilters";
+
+describe("buildSampleQueryFilters", () => {
+ it("hides internal evaluation environments without changing visible filters", () => {
+ const visibleFilters = [
+ {
+ column: "type",
+ type: "stringOptions" as const,
+ operator: "any of" as const,
+ value: ["GENERATION"],
+ },
+ ] satisfies FilterState;
+
+ const queryFilters = buildSampleQueryFilters(visibleFilters);
+
+ expect(queryFilters).toEqual([
+ ...visibleFilters,
+ {
+ column: "environment",
+ type: "stringOptions",
+ operator: "none of",
+ value: expect.arrayContaining([
+ "langfuse-llm-as-a-judge",
+ "langfuse-code-eval",
+ "langfuse-natural-language-filter",
+ "langfuse-prompt-experiment",
+ "langfuse-in-app-agent",
+ "langfuse-evaluation",
+ "llm-as-a-judge",
+ ]),
+ },
+ ]);
+ expect(visibleFilters).toHaveLength(1);
+ });
+
+ it("keeps valid experiment and user environment options visible", () => {
+ expect(
+ removeInternalEvaluationEnvironmentOptions({
+ environment: [
+ { value: "langfuse-code-eval" },
+ { value: "llm-as-a-judge" },
+ { value: "sdk-experiment" },
+ { value: "my-langfuse-production" },
+ { value: "production" },
+ ],
+ }),
+ ).toEqual({
+ environment: [
+ { value: "sdk-experiment" },
+ { value: "my-langfuse-production" },
+ { value: "production" },
+ ],
+ });
+ });
+});
diff --git a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/buildSampleQueryFilters.ts b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/buildSampleQueryFilters.ts
new file mode 100644
index 0000000000..6286c082bc
--- /dev/null
+++ b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/fns/buildSampleQueryFilters.ts
@@ -0,0 +1,34 @@
+import type { FilterState } from "@langfuse/shared";
+
+import type { ObservedOptions } from "@/src/features/search-bar/lib/observed-options";
+import {
+ INTERNAL_EVALUATION_ENVIRONMENTS,
+ INTERNAL_EVALUATION_ENVIRONMENT_FILTERS,
+} from "@/src/features/evals/v2/constants/experimentAndEvalFilters";
+
+export function buildSampleQueryFilters(
+ visibleFilters: FilterState,
+ additionalFilters: FilterState = [],
+): FilterState {
+ return [
+ ...visibleFilters,
+ ...INTERNAL_EVALUATION_ENVIRONMENT_FILTERS,
+ ...additionalFilters,
+ ];
+}
+
+export function removeInternalEvaluationEnvironmentOptions(
+ observed: ObservedOptions | undefined,
+): ObservedOptions | undefined {
+ if (observed?.environment === undefined) return observed;
+
+ const internalEnvironments = new Set<string>(
+ INTERNAL_EVALUATION_ENVIRONMENTS,
+ );
+ return {
+ ...observed,
+ environment: observed.environment.filter(
+ ({ value }) => !internalEnvironments.has(value),
+ ),
+ };
+}
diff --git a/web/src/features/evals/v2/components/Rules/CreateRuleDialog/components/CreateRuleDialogContent/CreateRuleDialogContent.tsx b/web/src/features/evals/v2/components/Rules/CreateRuleDialog/components/CreateRuleDialogContent/CreateRuleDialogContent.tsx
index 9d3cc81fc5..4b8eb67a68 100644
--- a/web/src/features/evals/v2/components/Rules/CreateRuleDialog/components/CreateRuleDialogContent/CreateRuleDialogContent.tsx
+++ b/web/src/features/evals/v2/components/Rules/CreateRuleDialog/components/CreateRuleDialogContent/CreateRuleDialogContent.tsx
@@ -19,14 +19,10 @@ import { showSuccessToast } from "@/src/features/notifications/showSuccessToast"
import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
import { api } from "@/src/utils/api";
import { trpcErrorToast } from "@/src/utils/trpcErrorToast";
-import { EXPERIMENTS_AND_EVALS_EXCLUSION_FILTERS } from "@/src/features/evals/v2/constants/experimentAndEvalFilters";
import { useLangfuseCloudRegion } from "@/src/features/organizations/hooks";
import { useProject } from "@/src/features/projects/hooks";
import { prepareNameForSave } from "@/src/features/evals/v2/fns/prepareNameForSave";
-
-function resolveInitialRuleFilters(initialFilter?: FilterState) {
- return initialFilter ?? EXPERIMENTS_AND_EVALS_EXCLUSION_FILTERS;
-}
+import { resolveInitialRuleFilters } from "./resolveInitialRuleFilters";
export function CreateRuleDialogContent({
projectId,
diff --git a/web/src/features/evals/v2/components/Rules/CreateRuleDialog/components/CreateRuleDialogContent/resolveInitialRuleFilters.clienttest.ts b/web/src/features/evals/v2/components/Rules/CreateRuleDialog/components/CreateRuleDialogContent/resolveInitialRuleFilters.clienttest.ts
new file mode 100644
index 0000000000..86074a12bb
--- /dev/null
+++ b/web/src/features/evals/v2/components/Rules/CreateRuleDialog/components/CreateRuleDialogContent/resolveInitialRuleFilters.clienttest.ts
@@ -0,0 +1,29 @@
+import { describe, expect, it } from "vitest";
+
+import { resolveInitialRuleFilters } from "./resolveInitialRuleFilters";
+
+describe("resolveInitialRuleFilters", () => {
+ it("starts new rules with a root span filter", () => {
+ expect(resolveInitialRuleFilters()).toEqual([
+ {
+ column: "isRootObservation",
+ type: "boolean",
+ operator: "=",
+ value: true,
+ },
+ ]);
+ });
+
+ it("preserves supplied filters", () => {
+ const filters = [
+ {
+ column: "type",
+ type: "stringOptions" as const,
+ operator: "any of" as const,
+ value: ["GENERATION"],
+ },
+ ];
+
+ expect(resolveInitialRuleFilters(filters)).toBe(filters);
+ });
+});
diff --git a/web/src/features/evals/v2/components/Rules/CreateRuleDialog/components/CreateRuleDialogContent/resolveInitialRuleFilters.ts b/web/src/features/evals/v2/components/Rules/CreateRuleDialog/components/CreateRuleDialogContent/resolveInitialRuleFilters.ts
new file mode 100644
index 0000000000..a445e9550c
--- /dev/null
+++ b/web/src/features/evals/v2/components/Rules/CreateRuleDialog/components/CreateRuleDialogContent/resolveInitialRuleFilters.ts
@@ -0,0 +1,16 @@
+import type { FilterState } from "@langfuse/shared";
+
+export function resolveInitialRuleFilters(
+ initialFilter?: FilterState,
+): FilterState {
+ return (
+ initialFilter ?? [
+ {
+ column: "isRootObservation",
+ type: "boolean",
+ operator: "=",
+ value: true,
+ },
+ ]
+ );
+}
diff --git a/web/src/features/evals/v2/constants/experimentAndEvalFilters.ts b/web/src/features/evals/v2/constants/experimentAndEvalFilters.ts
index eb3886986a..03e74106db 100644
--- a/web/src/features/evals/v2/constants/experimentAndEvalFilters.ts
+++ b/web/src/features/evals/v2/constants/experimentAndEvalFilters.ts
@@ -1,12 +1,26 @@
-import type { FilterState } from "@langfuse/shared";
+import {
+ LangfuseInternalTraceEnvironment,
+ type FilterState,
+} from "@langfuse/shared";
-export const EXPERIMENTS_AND_EVALS_EXCLUSION_FILTERS = [
+export const INTERNAL_EVALUATION_ENVIRONMENTS = [
+ ...Object.values(LangfuseInternalTraceEnvironment),
+ "langfuse-evaluation",
+ "langfuse",
+ "llm-as-a-judge",
+] as const;
+
+export const INTERNAL_EVALUATION_ENVIRONMENT_FILTERS = [
{
column: "environment",
- type: "string",
- operator: "does not contain",
- value: "langfuse-",
+ type: "stringOptions",
+ operator: "none of",
+ value: [...INTERNAL_EVALUATION_ENVIRONMENTS],
},
+] satisfies FilterState;
+
+export const EXPERIMENTS_AND_EVALS_EXCLUSION_FILTERS = [
+ ...INTERNAL_EVALUATION_ENVIRONMENT_FILTERS,
{
column: "environment",
type: "stringOptions",
diff --git a/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.clienttest.ts b/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.clienttest.ts
index e5b75f47d0..fd5d387976 100644
--- a/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.clienttest.ts
+++ b/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.clienttest.ts
@@ -2,7 +2,6 @@ import type { FilterState } from "@langfuse/shared";
import { describe, expect, it } from "vitest";
import { getDefaultCodeEvalSource } from "@/src/features/evals/utils/code-eval-template-starter-examples";
-import { EXPERIMENTS_AND_EVALS_EXCLUSION_FILTERS } from "@/src/features/evals/v2/constants/experimentAndEvalFilters";
import {
createEvaluatorSetupStore,
selectHasValidModel,
@@ -95,7 +94,6 @@ describe("createEvaluatorSetupStore", () => {
operator: "=",
value: true,
},
- ...EXPERIMENTS_AND_EVALS_EXCLUSION_FILTERS,
],
});
});
@@ -213,7 +211,6 @@ describe("createEvaluatorSetupStore", () => {
operator: "=",
value: true,
},
- ...EXPERIMENTS_AND_EVALS_EXCLUSION_FILTERS,
]);
});
diff --git a/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.ts b/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.ts
index 97f0bd0c0d..e45a31da38 100644
--- a/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.ts
+++ b/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.ts
@@ -21,7 +21,6 @@ import type { JudgeModel } from "@/src/features/evals/v2/judgeModel";
import type { ScoreOutputFormState } from "@/src/features/evals/v2/scoreOutputTypes";
import type { NormalizedEvaluatorDefinition } from "@/src/features/evals/v2/server/evaluators/evaluatorTypes";
import { toScoreOutputFormState } from "@/src/features/evals/v2/fns/scoreOutput/toScoreOutputFormState";
-import { EXPERIMENTS_AND_EVALS_EXCLUSION_FILTERS } from "@/src/features/evals/v2/constants/experimentAndEvalFilters";
import { safeRandomUUID } from "@/src/utils/safe-random-uuid";
const DEFAULT_PROMPT = `Evaluate the quality of the response.
@@ -197,7 +196,6 @@ export function createEvaluatorSetupStore({
selectedObservation: null,
sampleFilter: initialSampleFilter ?? [
...DEFAULT_OBSERVATION_FILTER_WHEN_REMAPPING,
- ...EXPERIMENTS_AND_EVALS_EXCLUSION_FILTERS,
],
promptPreviewEnabled: false,
testPanelOpen: true,