Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(evals): exclude internal eval environments from evaluator backfill (#18877)

Tobias Wochingerabc652fcfb7f
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/EvaluatorBackfillSettings.stories.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/EvaluatorBackfillSettings.stories.tsx
index 67bee2b7ea..26838e7ec7 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/EvaluatorBackfillSettings.stories.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/EvaluatorBackfillSettings.stories.tsx
@@ -51,6 +51,7 @@ const sharedArgs: EvaluatorBackfillSettingsProps = {
maxItems: 5_000,
maxAllowedItems: 25_000,
matchingObservations: 4_400,
+ sampling: 1,
isEstimating: false,
onEnabledChange: fn(),
onWindowChange: fn(),
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/EvaluatorBackfillSettings.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/EvaluatorBackfillSettings.tsx
index 3f4304b51c..301bdd943b 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/EvaluatorBackfillSettings.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/EvaluatorBackfillSettings.tsx
@@ -65,6 +65,7 @@ export function EvaluatorBackfillSettings({
maxItems,
maxAllowedItems,
matchingObservations,
+ sampling,
isEstimating,
onEnabledChange,
onWindowChange,
@@ -78,6 +79,7 @@ export function EvaluatorBackfillSettings({
maxItems: number;
maxAllowedItems: number;
matchingObservations: number;
+ sampling: number;
isEstimating: boolean;
onEnabledChange: (enabled: boolean) => void;
onWindowChange: (window: EvaluatorBackfillWindow) => void;
@@ -85,6 +87,10 @@ export function EvaluatorBackfillSettings({
onMaxItemsChange: (maxItems: number) => void;
}) {
const cappedCount = Math.min(matchingObservations, maxItems);
+ const samplingNote =
+ sampling < 1 && cappedCount > 0
+ ? ` At ${Math.round(sampling * 100)}% sampling, about ${numberFormatter(Math.round(cappedCount * sampling), 0)} will be scored.`
+ : "";
const rangeDays = Math.max(
1,
Math.ceil(
@@ -187,8 +193,8 @@ export function EvaluatorBackfillSettings({
{isEstimating
? "Counting matching observations..."
: matchingObservations > maxItems
- ? `${numberFormatter(matchingObservations, 0)} observations in ${rangeDescription} — capping at ${numberFormatter(cappedCount, 0)}, newest first.`
- : `${numberFormatter(matchingObservations, 0)} observations in ${rangeDescription}, all within your limit.`}
+ ? `${numberFormatter(matchingObservations, 0)} observations in ${rangeDescription} — capping at ${numberFormatter(cappedCount, 0)}, newest first.${samplingNote}`
+ : `${numberFormatter(matchingObservations, 0)} observations in ${rangeDescription}, all within your limit.${samplingNote}`}
</p>
</div>
) : null}
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedDialog.stories.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedDialog.stories.tsx
index d829f8ac4c..504701ee33 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedDialog.stories.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedDialog.stories.tsx
@@ -133,6 +133,7 @@ const sharedArgs = {
maxItems={5_000}
maxAllowedItems={25_000}
matchingObservations={4_400}
+ sampling={1}
isEstimating={false}
onEnabledChange={fn()}
onWindowChange={fn()}
@@ -241,6 +242,7 @@ export const WithBackfill = meta.story({
maxItems={5_000}
maxAllowedItems={25_000}
matchingObservations={4_400}
+ sampling={1}
isEstimating={false}
onEnabledChange={fn()}
onWindowChange={fn()}
@@ -269,6 +271,7 @@ export const WithCustomBackfill = meta.story({
maxItems={5_000}
maxAllowedItems={25_000}
matchingObservations={21_420}
+ sampling={1}
isEstimating={false}
onEnabledChange={fn()}
onWindowChange={fn()}
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsx
index 70b08249e9..625f3d2db2 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsx
@@ -537,6 +537,7 @@ export function EvaluatorSavedDialogContainer({
maxItems={backfill.effectiveMaxItems}
maxAllowedItems={backfill.allowedItems}
matchingObservations={backfill.matchingObservations}
+ sampling={currentBackfillScope?.sampling ?? 1}
isEstimating={backfill.isEstimating}
onEnabledChange={(enabled) => {
onboardingAnalytics?.track("eval:onboarding_historic_eval_toggled", {
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/hooks/useEvaluatorSavedBackfill.clienttest.ts b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/hooks/useEvaluatorSavedBackfill.clienttest.ts
new file mode 100644
index 0000000000..f8b7a4a48a
--- /dev/null
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/hooks/useEvaluatorSavedBackfill.clienttest.ts
@@ -0,0 +1,96 @@
+import { act, renderHook } from "@testing-library/react";
+import { describe, expect, it, vi } from "vitest";
+import { LangfuseInternalTraceEnvironment } from "@langfuse/shared";
+import { useEvaluatorSavedBackfill } from "./useEvaluatorSavedBackfill";
+
+const mocks = vi.hoisted(() => ({
+ estimate: vi
+ .fn()
+ .mockResolvedValue([
+ { evaluatorId: "evaluator-1", matchingObservations: 20 },
+ ]),
+ runEvaluation: vi.fn().mockResolvedValue({ id: "batch-action-1" }),
+}));
+
+vi.mock("@/src/utils/api", () => ({
+ api: {
+ useUtils: () => ({
+ client: {
+ evalsV2: {
+ activationCostEstimates: { mutate: mocks.estimate },
+ },
+ },
+ }),
+ batchAction: {
+ runEvaluation: {
+ create: {
+ useMutation: () => ({
+ mutateAsync: mocks.runEvaluation,
+ isPending: false,
+ }),
+ },
+ },
+ },
+ },
+}));
+
+vi.mock("@/src/utils/trpcErrorToast", () => ({
+ trpcErrorToast: vi.fn(),
+}));
+
+const excludedEnvironments = (filter: unknown[]) =>
+ filter.flatMap((condition) => {
+ const { column, operator, value } = condition as {
+ column: string;
+ operator: string;
+ value: unknown;
+ };
+ return column === "environment" && operator === "none of"
+ ? (value as string[])
+ : [];
+ });
+
+describe("useEvaluatorSavedBackfill", () => {
+ it("excludes the environments eval execution rejects from the estimate and the backfill run", async () => {
+ const scope = {
+ filter: [
+ {
+ column: "type",
+ type: "stringOptions" as const,
+ operator: "any of" as const,
+ value: ["GENERATION"],
+ },
+ ],
+ sampling: 1,
+ };
+ const { result } = renderHook(() =>
+ useEvaluatorSavedBackfill({
+ projectId: "project-1",
+ evaluatorId: "evaluator-1",
+ claimMissingCostTest: () => false,
+ resetMissingCostTest: vi.fn(),
+ }),
+ );
+
+ await act(async () => result.current.setEnabled(true, scope));
+ await act(() =>
+ result.current.schedule(scope, result.current.executionRange),
+ );
+
+ const estimateFilter = mocks.estimate.mock.calls[0]![0].filter;
+ const runFilter = mocks.runEvaluation.mock.calls[0]![0].query.filter;
+ for (const filter of [estimateFilter, runFilter]) {
+ expect(filter).toEqual(expect.arrayContaining(scope.filter));
+ expect(excludedEnvironments(filter)).toEqual(
+ expect.arrayContaining([
+ LangfuseInternalTraceEnvironment.LLMJudge,
+ LangfuseInternalTraceEnvironment.CodeEval,
+ "llm-as-a-judge",
+ ]),
+ );
+ expect(excludedEnvironments(filter)).not.toContain(
+ LangfuseInternalTraceEnvironment.PromptExperiments,
+ );
+ }
+ });
+});
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/hooks/useEvaluatorSavedBackfill.ts b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/hooks/useEvaluatorSavedBackfill.ts
index 73c3807864..9dec5e85d4 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/hooks/useEvaluatorSavedBackfill.ts
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/hooks/useEvaluatorSavedBackfill.ts
@@ -1,4 +1,8 @@
-import { BatchEvalSourceTable, type FilterState } from "@langfuse/shared";
+import {
+ BatchEvalSourceTable,
+ LangfuseInternalTraceEnvironment,
+ type FilterState,
+} from "@langfuse/shared";
import { endOfDay, startOfDay, subDays, subHours, subMonths } from "date-fns";
import { useCallback, useRef, useState } from "react";
import type {
@@ -9,6 +13,7 @@ import {
DEFAULT_EVALUATOR_BACKFILL_ITEMS,
MAX_EVALUATOR_BACKFILL_ITEMS,
} from "@/src/features/evals/v2/constants/evaluatorBackfill";
+import { INTERNAL_EVALUATION_ENVIRONMENTS } from "@/src/features/evals/v2/constants/experimentAndEvalFilters";
import { api } from "@/src/utils/api";
import { trpcErrorToast } from "@/src/utils/trpcErrorToast";
@@ -17,6 +22,19 @@ type EvaluatorBackfillScope = {
sampling: number;
};
+/** Excludes environments eval execution rejects; prompt-experiment outputs stay eligible */
+const BACKFILL_ENVIRONMENT_FILTERS: FilterState = [
+ {
+ column: "environment",
+ type: "stringOptions",
+ operator: "none of",
+ value: INTERNAL_EVALUATION_ENVIRONMENTS.filter(
+ (environment) =>
+ environment !== LangfuseInternalTraceEnvironment.PromptExperiments,
+ ),
+ },
+];
+
function getBackfillRange(
window: Exclude<EvaluatorBackfillWindow, "custom">,
now = new Date(),
@@ -102,7 +120,7 @@ export function useEvaluatorSavedBackfill({
await utils.client.evalsV2.activationCostEstimates.mutate({
projectId,
evaluatorIds: [evaluatorId],
- filter: scope.filter,
+ filter: [...scope.filter, ...BACKFILL_ENVIRONMENT_FILTERS],
sampling: scope.sampling,
shouldRunMissingTest,
timeRange: estimateRange,
@@ -219,6 +237,7 @@ export function useEvaluatorSavedBackfill({
query: {
filter: [
...scope.filter,
+ ...BACKFILL_ENVIRONMENT_FILTERS,
{
column: "startTime",
type: "datetime",