🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(evals): exclude internal eval environments from evaluator backfill (#18877)
Tobias Wochingerabc652fcfb7f
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/EvaluatorBackfillSettings.stories.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/EvaluatorBackfillSettings.stories.tsxindex 67bee2b7ea..26838e7ec7 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/EvaluatorBackfillSettings.stories.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/EvaluatorBackfillSettings.stories.tsx@@ -51,6 +51,7 @@ const sharedArgs: EvaluatorBackfillSettingsProps = {maxItems: 5_000,maxAllowedItems: 25_000,matchingObservations: 4_400,+ sampling: 1,isEstimating: false,onEnabledChange: fn(),onWindowChange: fn(),diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/EvaluatorBackfillSettings.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/EvaluatorBackfillSettings.tsxindex 3f4304b51c..301bdd943b 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/EvaluatorBackfillSettings.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/EvaluatorBackfillSettings.tsx@@ -65,6 +65,7 @@ export function EvaluatorBackfillSettings({maxItems,maxAllowedItems,matchingObservations,+ sampling,isEstimating,onEnabledChange,onWindowChange,@@ -78,6 +79,7 @@ export function EvaluatorBackfillSettings({maxItems: number;maxAllowedItems: number;matchingObservations: number;+ sampling: number;isEstimating: boolean;onEnabledChange: (enabled: boolean) => void;onWindowChange: (window: EvaluatorBackfillWindow) => void;@@ -85,6 +87,10 @@ export function EvaluatorBackfillSettings({onMaxItemsChange: (maxItems: number) => void;}) {const cappedCount = Math.min(matchingObservations, maxItems);+ const samplingNote =+ sampling < 1 && cappedCount > 0+ ? ` At ${Math.round(sampling * 100)}% sampling, about ${numberFormatter(Math.round(cappedCount * sampling), 0)} will be scored.`+ : "";const rangeDays = Math.max(1,Math.ceil(@@ -187,8 +193,8 @@ export function EvaluatorBackfillSettings({{isEstimating? "Counting matching observations...": matchingObservations > maxItems- ? `${numberFormatter(matchingObservations, 0)} observations in ${rangeDescription} — capping at ${numberFormatter(cappedCount, 0)}, newest first.`- : `${numberFormatter(matchingObservations, 0)} observations in ${rangeDescription}, all within your limit.`}+ ? `${numberFormatter(matchingObservations, 0)} observations in ${rangeDescription} — capping at ${numberFormatter(cappedCount, 0)}, newest first.${samplingNote}`+ : `${numberFormatter(matchingObservations, 0)} observations in ${rangeDescription}, all within your limit.${samplingNote}`}</p></div>) : null}diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedDialog.stories.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedDialog.stories.tsxindex d829f8ac4c..504701ee33 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedDialog.stories.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedDialog.stories.tsx@@ -133,6 +133,7 @@ const sharedArgs = {maxItems={5_000}maxAllowedItems={25_000}matchingObservations={4_400}+ sampling={1}isEstimating={false}onEnabledChange={fn()}onWindowChange={fn()}@@ -241,6 +242,7 @@ export const WithBackfill = meta.story({maxItems={5_000}maxAllowedItems={25_000}matchingObservations={4_400}+ sampling={1}isEstimating={false}onEnabledChange={fn()}onWindowChange={fn()}@@ -269,6 +271,7 @@ export const WithCustomBackfill = meta.story({maxItems={5_000}maxAllowedItems={25_000}matchingObservations={21_420}+ sampling={1}isEstimating={false}onEnabledChange={fn()}onWindowChange={fn()}diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsxindex 70b08249e9..625f3d2db2 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsx@@ -537,6 +537,7 @@ export function EvaluatorSavedDialogContainer({maxItems={backfill.effectiveMaxItems}maxAllowedItems={backfill.allowedItems}matchingObservations={backfill.matchingObservations}+ sampling={currentBackfillScope?.sampling ?? 1}isEstimating={backfill.isEstimating}onEnabledChange={(enabled) => {onboardingAnalytics?.track("eval:onboarding_historic_eval_toggled", {diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/hooks/useEvaluatorSavedBackfill.clienttest.ts b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/hooks/useEvaluatorSavedBackfill.clienttest.tsnew file mode 100644index 0000000000..f8b7a4a48a--- /dev/null+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/hooks/useEvaluatorSavedBackfill.clienttest.ts@@ -0,0 +1,96 @@+import { act, renderHook } from "@testing-library/react";+import { describe, expect, it, vi } from "vitest";+import { LangfuseInternalTraceEnvironment } from "@langfuse/shared";+import { useEvaluatorSavedBackfill } from "./useEvaluatorSavedBackfill";++const mocks = vi.hoisted(() => ({+ estimate: vi+ .fn()+ .mockResolvedValue([+ { evaluatorId: "evaluator-1", matchingObservations: 20 },+ ]),+ runEvaluation: vi.fn().mockResolvedValue({ id: "batch-action-1" }),+}));++vi.mock("@/src/utils/api", () => ({+ api: {+ useUtils: () => ({+ client: {+ evalsV2: {+ activationCostEstimates: { mutate: mocks.estimate },+ },+ },+ }),+ batchAction: {+ runEvaluation: {+ create: {+ useMutation: () => ({+ mutateAsync: mocks.runEvaluation,+ isPending: false,+ }),+ },+ },+ },+ },+}));++vi.mock("@/src/utils/trpcErrorToast", () => ({+ trpcErrorToast: vi.fn(),+}));++const excludedEnvironments = (filter: unknown[]) =>+ filter.flatMap((condition) => {+ const { column, operator, value } = condition as {+ column: string;+ operator: string;+ value: unknown;+ };+ return column === "environment" && operator === "none of"+ ? (value as string[])+ : [];+ });++describe("useEvaluatorSavedBackfill", () => {+ it("excludes the environments eval execution rejects from the estimate and the backfill run", async () => {+ const scope = {+ filter: [+ {+ column: "type",+ type: "stringOptions" as const,+ operator: "any of" as const,+ value: ["GENERATION"],+ },+ ],+ sampling: 1,+ };+ const { result } = renderHook(() =>+ useEvaluatorSavedBackfill({+ projectId: "project-1",+ evaluatorId: "evaluator-1",+ claimMissingCostTest: () => false,+ resetMissingCostTest: vi.fn(),+ }),+ );++ await act(async () => result.current.setEnabled(true, scope));+ await act(() =>+ result.current.schedule(scope, result.current.executionRange),+ );++ const estimateFilter = mocks.estimate.mock.calls[0]![0].filter;+ const runFilter = mocks.runEvaluation.mock.calls[0]![0].query.filter;+ for (const filter of [estimateFilter, runFilter]) {+ expect(filter).toEqual(expect.arrayContaining(scope.filter));+ expect(excludedEnvironments(filter)).toEqual(+ expect.arrayContaining([+ LangfuseInternalTraceEnvironment.LLMJudge,+ LangfuseInternalTraceEnvironment.CodeEval,+ "llm-as-a-judge",+ ]),+ );+ expect(excludedEnvironments(filter)).not.toContain(+ LangfuseInternalTraceEnvironment.PromptExperiments,+ );+ }+ });+});diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/hooks/useEvaluatorSavedBackfill.ts b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/hooks/useEvaluatorSavedBackfill.tsindex 73c3807864..9dec5e85d4 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/hooks/useEvaluatorSavedBackfill.ts+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/hooks/useEvaluatorSavedBackfill.ts@@ -1,4 +1,8 @@-import { BatchEvalSourceTable, type FilterState } from "@langfuse/shared";+import {+ BatchEvalSourceTable,+ LangfuseInternalTraceEnvironment,+ type FilterState,+} from "@langfuse/shared";import { endOfDay, startOfDay, subDays, subHours, subMonths } from "date-fns";import { useCallback, useRef, useState } from "react";import type {@@ -9,6 +13,7 @@ import {DEFAULT_EVALUATOR_BACKFILL_ITEMS,MAX_EVALUATOR_BACKFILL_ITEMS,} from "@/src/features/evals/v2/constants/evaluatorBackfill";+import { INTERNAL_EVALUATION_ENVIRONMENTS } from "@/src/features/evals/v2/constants/experimentAndEvalFilters";import { api } from "@/src/utils/api";import { trpcErrorToast } from "@/src/utils/trpcErrorToast";@@ -17,6 +22,19 @@ type EvaluatorBackfillScope = {sampling: number;};+/** Excludes environments eval execution rejects; prompt-experiment outputs stay eligible */+const BACKFILL_ENVIRONMENT_FILTERS: FilterState = [+ {+ column: "environment",+ type: "stringOptions",+ operator: "none of",+ value: INTERNAL_EVALUATION_ENVIRONMENTS.filter(+ (environment) =>+ environment !== LangfuseInternalTraceEnvironment.PromptExperiments,+ ),+ },+];+function getBackfillRange(window: Exclude<EvaluatorBackfillWindow, "custom">,now = new Date(),@@ -102,7 +120,7 @@ export function useEvaluatorSavedBackfill({await utils.client.evalsV2.activationCostEstimates.mutate({projectId,evaluatorIds: [evaluatorId],- filter: scope.filter,+ filter: [...scope.filter, ...BACKFILL_ENVIRONMENT_FILTERS],sampling: scope.sampling,shouldRunMissingTest,timeRange: estimateRange,@@ -219,6 +237,7 @@ export function useEvaluatorSavedBackfill({query: {filter: [...scope.filter,+ ...BACKFILL_ENVIRONMENT_FILTERS,{column: "startTime",type: "datetime",