🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(evals): backfill saved evaluators (#17071)
Tobias Wochingerc1b1b899b785
diff --git a/packages/shared/src/server/queues.ts b/packages/shared/src/server/queues.tsindex 6c87ba44f2..7d995ea025 100644--- a/packages/shared/src/server/queues.ts+++ b/packages/shared/src/server/queues.ts@@ -269,6 +269,8 @@ export const BatchActionProcessingEventSchema = z.discriminatedUnion(evaluatorIds: z.array(z.string()),evalVersion: z.literal("v2").optional(),evaluatorMappings: z.array(BatchEvalEvaluatorMappingSchema).optional(),+ sampling: z.number().min(0).max(1).optional(),+ rowLimit: z.number().int().positive().optional(),}),],);diff --git a/web/src/__tests__/server/unit/batch-actions-comment-filter.servertest.ts b/web/src/__tests__/server/unit/batch-actions-comment-filter.servertest.tsindex 447a2390f7..f8439c3a9f 100644--- a/web/src/__tests__/server/unit/batch-actions-comment-filter.servertest.ts+++ b/web/src/__tests__/server/unit/batch-actions-comment-filter.servertest.ts@@ -317,6 +317,32 @@ describe("batched evaluation version selection", () => {);});+ it("allows a capped batch when more observations match", async () => {+ const context = prepare({ v4BetaEnabled: true });+ mocks.getObservationsCountFromEventsTable.mockResolvedValue(100_000);++ await context.runEvaluation.create({+ projectId,+ query,+ evaluatorIds: [evaluatorId],+ sourceTable: BatchEvalSourceTable.EVENTS,+ evalVersion: "v2",+ sampling: 0.25,+ rowLimit: 5_000,+ });++ expect(mocks.queueAdd).toHaveBeenCalledWith(+ expect.anything(),+ expect.objectContaining({+ payload: expect.objectContaining({+ sampling: 0.25,+ rowLimit: 5_000,+ }),+ }),+ expect.anything(),+ );+ });+it("rejects evaluator v2 for users outside fast preview", async () => {const context = prepare();@@ -374,6 +400,8 @@ describe("batched evaluation version selection", () => {sourceTable: BatchEvalSourceTable.EVENTS,evalVersion: "v2",evaluatorMappings,+ sampling: 0.25,+ rowLimit: 5_000,});expect(context.prisma.evaluator.findMany).toHaveBeenCalledWith(@@ -390,6 +418,8 @@ describe("batched evaluation version selection", () => {config: expect.objectContaining({evalVersion: "v2",evaluatorMappings,+ sampling: 0.25,+ rowLimit: 5_000,}),}),}),@@ -400,6 +430,8 @@ describe("batched evaluation version selection", () => {payload: expect.objectContaining({evalVersion: "v2",evaluatorMappings,+ sampling: 0.25,+ rowLimit: 5_000,}),}),expect.anything(),diff --git a/web/src/features/batch-actions/server/runEvaluationRouter.ts b/web/src/features/batch-actions/server/runEvaluationRouter.tsindex 5e860abcc8..b75675cf90 100644--- a/web/src/features/batch-actions/server/runEvaluationRouter.ts+++ b/web/src/features/batch-actions/server/runEvaluationRouter.ts@@ -42,6 +42,8 @@ export const runEvaluationRouter = createTRPCRouter({evaluatorIds: rawEvaluatorIds,sourceTable = BatchEvalSourceTable.EVENTS,evaluatorMappings: rawEvaluatorMappings,+ sampling,+ rowLimit,} = input;if (env.LANGFUSE_MIGRATION_V4_ALLOW_PREVIEW_OPT_IN !== "true") {@@ -145,7 +147,10 @@ export const runEvaluationRouter = createTRPCRouter({? 0: await getObservationsCountFromEventsTable(countQueryOpts);- if (observationCount > env.LANGFUSE_MAX_HISTORIC_EVAL_CREATION_LIMIT) {+ if (+ rowLimit === undefined &&+ observationCount > env.LANGFUSE_MAX_HISTORIC_EVAL_CREATION_LIMIT+ ) {throw new TRPCError({code: "BAD_REQUEST",message: `Too many observations selected. Maximum allowed is ${env.LANGFUSE_MAX_HISTORIC_EVAL_CREATION_LIMIT}, but ${observationCount} observations match your filters. Please refine your filters to reduce the count.`,@@ -157,6 +162,8 @@ export const runEvaluationRouter = createTRPCRouter({evaluatorIds,...(input.evalVersion ? { evalVersion: input.evalVersion } : {}),...(evaluatorMappings ? { evaluatorMappings } : {}),+ ...(sampling !== undefined ? { sampling } : {}),+ ...(rowLimit !== undefined ? { rowLimit } : {}),};logger.info(@@ -206,6 +213,8 @@ export const runEvaluationRouter = createTRPCRouter({? { evalVersion: batchConfig.evalVersion }: {}),...(evaluatorMappings ? { evaluatorMappings } : {}),+ ...(sampling !== undefined ? { sampling } : {}),+ ...(rowLimit !== undefined ? { rowLimit } : {}),},},{diff --git a/web/src/features/batch-actions/validation.ts b/web/src/features/batch-actions/validation.tsindex 96f8ce1b14..d4c8664a65 100644--- a/web/src/features/batch-actions/validation.ts+++ b/web/src/features/batch-actions/validation.ts@@ -26,6 +26,8 @@ export const CreateObservationBatchEvaluationActionSchema = z.array(BatchEvalEvaluatorMappingSchema).max(BATCH_EVAL_EVALUATOR_LIMIT).optional(),+ sampling: z.number().min(0).max(1).optional(),+ rowLimit: z.number().int().positive().max(25_000).optional(),}).superRefine((value, ctx) => {if (!value.evaluatorMappings) return;diff --git a/web/src/features/evals/v2/components/EvaluatorCostCalculationTooltipContent/EvaluatorCostCalculationTooltipContent.tsx b/web/src/features/evals/v2/components/EvaluatorCostCalculationTooltipContent/EvaluatorCostCalculationTooltipContent.tsxnew file mode 100644index 0000000000..db6385c071--- /dev/null+++ b/web/src/features/evals/v2/components/EvaluatorCostCalculationTooltipContent/EvaluatorCostCalculationTooltipContent.tsx@@ -0,0 +1,18 @@+export function EvaluatorCostCalculationTooltipContent({+ calculation,+ explanation,+}: {+ calculation: string | null;+ explanation: string;+}) {+ return (+ <div className="space-y-2">+ {calculation ? (+ <p className="font-mono tabular-nums">{calculation}</p>+ ) : null}+ <p className={calculation ? "text-muted-foreground" : undefined}>+ {explanation}+ </p>+ </div>+ );+}diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/EvaluatorBackfillSettings.stories.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/EvaluatorBackfillSettings.stories.tsxnew file mode 100644index 0000000000..67bee2b7ea--- /dev/null+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/EvaluatorBackfillSettings.stories.tsx@@ -0,0 +1,85 @@+import { useArgs } from "storybook/preview-api";+import { fn } from "storybook/test";++import preview from "../../../../../../../.storybook/preview";+import { EvaluatorBackfillSettings } from "./EvaluatorBackfillSettings";++const meta = preview.meta({ component: EvaluatorBackfillSettings });++type EvaluatorBackfillSettingsProps = Parameters<+ typeof EvaluatorBackfillSettings+>[0];++function StatefulEvaluatorBackfillSettings(+ args: EvaluatorBackfillSettingsProps,+) {+ const [, updateArgs] = useArgs<EvaluatorBackfillSettingsProps>();++ return (+ <div className="max-w-xl rounded-md border p-3">+ <EvaluatorBackfillSettings+ {...args}+ onEnabledChange={(enabled) => {+ updateArgs({ enabled });+ args.onEnabledChange(enabled);+ }}+ onWindowChange={(selectedWindow) => {+ updateArgs({ selectedWindow });+ args.onWindowChange(selectedWindow);+ }}+ onRangeChange={(range) => {+ updateArgs({ range });+ args.onRangeChange(range);+ }}+ onMaxItemsChange={(maxItems) => {+ updateArgs({ maxItems });+ args.onMaxItemsChange(maxItems);+ }}+ />+ </div>+ );+}++const sharedArgs: EvaluatorBackfillSettingsProps = {+ enabled: true,+ canEnable: true,+ selectedWindow: "7-days",+ range: {+ from: new Date("2026-08-31T00:00:00"),+ to: new Date("2026-09-07T23:59:59.999"),+ },+ maxItems: 5_000,+ maxAllowedItems: 25_000,+ matchingObservations: 4_400,+ isEstimating: false,+ onEnabledChange: fn(),+ onWindowChange: fn(),+ onRangeChange: fn(),+ onMaxItemsChange: fn(),+};++export const LastSevenDays = meta.story({+ args: sharedArgs,+ render: StatefulEvaluatorBackfillSettings,+});++export const CustomRange = meta.story({+ args: {+ ...sharedArgs,+ selectedWindow: "custom",+ range: {+ from: new Date("2026-08-01T00:00:00"),+ to: new Date("2026-09-07T23:59:59.999"),+ },+ matchingObservations: 21_420,+ },+ render: StatefulEvaluatorBackfillSettings,+});++export const Disabled = meta.story({+ args: {+ ...sharedArgs,+ enabled: false,+ },+ render: StatefulEvaluatorBackfillSettings,+});diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/EvaluatorBackfillSettings.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/EvaluatorBackfillSettings.tsxnew file mode 100644index 0000000000..58762d20c9--- /dev/null+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/EvaluatorBackfillSettings.tsx@@ -0,0 +1,196 @@+import { format, subMonths } from "date-fns";+import { Switch } from "@/src/components/design-system/Switch/Switch";+import { DateRangeInput } from "@/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/components/DateRangeInput/DateRangeInput";+import { Input } from "@/src/components/ui/input";+import {+ Select,+ SelectContent,+ SelectItem,+ SelectTrigger,+ SelectValue,+} from "@/src/components/ui/select";+import { numberFormatter } from "@/src/utils/numbers";+import { cn } from "@/src/utils/tailwind";++export type EvaluatorBackfillWindow =+ | "24-hours"+ | "7-days"+ | "30-days"+ | "90-days"+ | "custom";++export type EvaluatorBackfillRange = {+ from: Date;+ to: Date;+};++const windowOptions: Array<{+ value: EvaluatorBackfillWindow;+ label: string;+ summaryLabel: string;+}> = [+ {+ value: "24-hours",+ label: "Last 24 hours",+ summaryLabel: "the last 24 hours",+ },+ {+ value: "7-days",+ label: "Last 7 days",+ summaryLabel: "the last 7 days",+ },+ {+ value: "30-days",+ label: "Last 30 days",+ summaryLabel: "the last 30 days",+ },+ {+ value: "90-days",+ label: "Last 90 days",+ summaryLabel: "the last 90 days",+ },+ {+ value: "custom",+ label: "Custom range",+ summaryLabel: "the selected range",+ },+];++export function EvaluatorBackfillSettings({+ enabled,+ canEnable,+ selectedWindow,+ range,+ maxItems,+ maxAllowedItems,+ matchingObservations,+ isEstimating,+ onEnabledChange,+ onWindowChange,+ onRangeChange,+ onMaxItemsChange,+}: {+ enabled: boolean;+ canEnable: boolean;+ selectedWindow: EvaluatorBackfillWindow;+ range: EvaluatorBackfillRange;+ maxItems: number;+ maxAllowedItems: number;+ matchingObservations: number;+ isEstimating: boolean;+ onEnabledChange: (enabled: boolean) => void;+ onWindowChange: (window: EvaluatorBackfillWindow) => void;+ onRangeChange: (range: EvaluatorBackfillRange) => void;+ onMaxItemsChange: (maxItems: number) => void;+}) {+ const cappedCount = Math.min(matchingObservations, maxItems);+ const rangeDays = Math.max(+ 1,+ Math.ceil(+ (range.to.getTime() - range.from.getTime()) / (24 * 60 * 60 * 1_000),+ ),+ );+ const selectedWindowOption = windowOptions.find(+ (option) => option.value === selectedWindow,+ );+ const rangeDescription =+ selectedWindow === "custom"+ ? `the selected range (${rangeDays} days)`+ : (selectedWindowOption?.summaryLabel ?? "the selected range");++ return (+ <section className="ml-6 max-w-full min-w-0">+ <div className="flex min-w-0 items-center gap-2">+ <Switch+ id="evaluator-backfill-enabled"+ size="sm"+ checked={enabled}+ disabled={!canEnable}+ onCheckedChange={onEnabledChange}+ />+ <label+ htmlFor="evaluator-backfill-enabled"+ className={cn(+ "min-w-0 text-sm leading-none",+ canEnable ? "cursor-pointer" : "text-muted-foreground",+ )}+ >+ Run evaluator on past observations matching these filters+ </label>+ </div>++ {enabled ? (+ <div className="mt-3 ml-9 max-w-full min-w-0 space-y-2">+ <div className="flex min-w-0 flex-wrap items-center gap-2 text-sm">+ <span>Score the newest</span>+ <Input+ id="evaluator-backfill-max-items"+ aria-label="Max items"+ type="number"+ min={1}+ max={maxAllowedItems}+ className="w-20 text-right font-mono text-sm"+ value={maxItems}+ onChange={(event) => {+ const value = Number(event.target.value);+ if (!Number.isFinite(value)) return;+ onMaxItemsChange(+ Math.min(maxAllowedItems, Math.max(1, Math.floor(value))),+ );+ }}+ />+ <span>matches from the</span>+ <Select+ value={selectedWindow}+ onValueChange={(value) =>+ onWindowChange(value as EvaluatorBackfillWindow)+ }+ >+ <SelectTrigger+ aria-label="Backfill time window"+ className="w-fit max-w-full whitespace-nowrap"+ disableValueLineClamp+ >+ <SelectValue />+ </SelectTrigger>+ <SelectContent>+ {windowOptions.map((option) => (+ <SelectItem key={option.value} value={option.value}>+ {option.label}+ </SelectItem>+ ))}+ </SelectContent>+ </Select>+ </div>++ {selectedWindow === "custom" ? (+ <DateRangeInput+ value={{+ from: format(range.from, "yyyy-MM-dd"),+ to: format(range.to, "yyyy-MM-dd"),+ }}+ min={format(subMonths(new Date(), 6), "yyyy-MM-dd")}+ max={format(new Date(), "yyyy-MM-dd")}+ fromAriaLabel="Backfill start date"+ toAriaLabel="Backfill end date"+ onValueChange={(value) =>+ onRangeChange({+ from: new Date(`${value.from}T00:00:00`),+ to: new Date(`${value.to}T23:59:59.999`),+ })+ }+ />+ ) : null}++ <p className="text-muted-foreground text-xs break-words">+ {isEstimating+ ? "Counting matching observations..."+ : matchingObservations > maxItems+ ? `${numberFormatter(matchingObservations, 0)} observations in ${rangeDescription} — capping at ${numberFormatter(cappedCount, 0)}, newest first.`+ : `${numberFormatter(matchingObservations, 0)} observations in ${rangeDescription}, all within your limit.`}+ </p>+ </div>+ ) : null}+ </section>+ );+}diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/components/DateRangeInput/DateRangeInput.clienttest.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/components/DateRangeInput/DateRangeInput.clienttest.tsxnew file mode 100644index 0000000000..236771097a--- /dev/null+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/components/DateRangeInput/DateRangeInput.clienttest.tsx@@ -0,0 +1,59 @@+import { fireEvent, render, screen } from "@testing-library/react";++import { DateRangeInput } from "./DateRangeInput";++const value = {+ from: "2026-08-31",+ to: "2026-09-07",+};++describe("DateRangeInput", () => {+ it("renders compact bounded date inputs", () => {+ render(+ <DateRangeInput+ value={value}+ min="2026-03-07"+ max="2026-09-07"+ onValueChange={vi.fn()}+ />,+ );++ expect(screen.getByLabelText("Start date")).toHaveAttribute(+ "min",+ "2026-03-07",+ );+ expect(screen.getByLabelText("Start date")).toHaveAttribute(+ "max",+ value.to,+ );+ expect(screen.getByLabelText("End date")).toHaveAttribute(+ "min",+ value.from,+ );+ expect(screen.getByLabelText("End date")).toHaveAttribute(+ "max",+ "2026-09-07",+ );+ });++ it("emits the complete range when either date changes", () => {+ const onValueChange = vi.fn();+ render(<DateRangeInput value={value} onValueChange={onValueChange} />);++ fireEvent.change(screen.getByLabelText("Start date"), {+ target: { value: "2026-08-30" },+ });+ expect(onValueChange).toHaveBeenLastCalledWith({+ from: "2026-08-30",+ to: value.to,+ });++ fireEvent.change(screen.getByLabelText("End date"), {+ target: { value: "2026-09-06" },+ });+ expect(onValueChange).toHaveBeenLastCalledWith({+ from: value.from,+ to: "2026-09-06",+ });+ });+});diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/components/DateRangeInput/DateRangeInput.stories.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/components/DateRangeInput/DateRangeInput.stories.tsxnew file mode 100644index 0000000000..6053fda10e--- /dev/null+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/components/DateRangeInput/DateRangeInput.stories.tsx@@ -0,0 +1,59 @@+import { useArgs } from "storybook/preview-api";+import { fn } from "storybook/test";++import preview from "../../../../../../../../../.storybook/preview";+import { DateRangeInput } from "./DateRangeInput";++const meta = preview.meta({ component: DateRangeInput });++type DateRangeInputProps = Parameters<typeof DateRangeInput>[0];++function StatefulDateRangeInput(args: DateRangeInputProps) {+ const [, updateArgs] = useArgs<DateRangeInputProps>();++ return (+ <DateRangeInput+ {...args}+ onValueChange={(value) => {+ updateArgs({ value });+ args.onValueChange(value);+ }}+ />+ );+}++export const Default = meta.story({+ args: {+ value: {+ from: "2026-08-31",+ to: "2026-09-07",+ },+ onValueChange: fn(),+ },+ render: StatefulDateRangeInput,+});++export const Bounded = meta.story({+ args: {+ value: {+ from: "2026-08-31",+ to: "2026-09-07",+ },+ min: "2026-03-07",+ max: "2026-09-07",+ onValueChange: fn(),+ },+ render: StatefulDateRangeInput,+});++export const Disabled = meta.story({+ args: {+ value: {+ from: "2026-08-31",+ to: "2026-09-07",+ },+ disabled: true,+ onValueChange: fn(),+ },+ render: StatefulDateRangeInput,+});diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/components/DateRangeInput/DateRangeInput.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/components/DateRangeInput/DateRangeInput.tsxnew file mode 100644index 0000000000..bf3c2039d0--- /dev/null+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/components/DateRangeInput/DateRangeInput.tsx@@ -0,0 +1,58 @@+import { Input } from "@/src/components/ui/input";++export type DateInputRange = {+ from: string;+ to: string;+};++export function DateRangeInput({+ value,+ onValueChange,+ min,+ max,+ fromAriaLabel = "Start date",+ toAriaLabel = "End date",+ disabled,+}: {+ value: DateInputRange;+ onValueChange: (value: DateInputRange) => void;+ min?: string;+ max?: string;+ fromAriaLabel?: string;+ toAriaLabel?: string;+ disabled?: boolean;+}) {+ return (+ <div className="grid w-fit max-w-full grid-cols-[minmax(0,7.5rem)_auto_minmax(0,7.5rem)] items-center gap-2">+ <Input+ aria-label={fromAriaLabel}+ type="date"+ className="block min-w-0 [&::-webkit-calendar-picker-indicator]:translate-y-px"+ value={value.from}+ min={min}+ max={value.to}+ disabled={disabled}+ onChange={(event) => {+ if (!event.target.value) return;+ onValueChange({ ...value, from: event.target.value });+ }}+ />+ <span className="text-muted-foreground" aria-hidden="true">+ →+ </span>+ <Input+ aria-label={toAriaLabel}+ type="date"+ className="block min-w-0 [&::-webkit-calendar-picker-indicator]:translate-y-px"+ value={value.to}+ min={value.from}+ max={max}+ disabled={disabled}+ onChange={(event) => {+ if (!event.target.value) return;+ onValueChange({ ...value, to: event.target.value });+ }}+ />+ </div>+ );+}diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedCostSummary.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedCostSummary.tsxindex 544f4fa9fc..80e9debe2d 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedCostSummary.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedCostSummary.tsx@@ -6,6 +6,7 @@ import {SAMPLING_SLIDER_MIN,SAMPLING_SLIDER_STEP,} from "@/src/features/evals/v2/constants/ruleSampling";+import { EvaluatorCostCalculationTooltipContent } from "@/src/features/evals/v2/components/EvaluatorCostCalculationTooltipContent/EvaluatorCostCalculationTooltipContent";import type { ActivationEstimate } from "@/src/features/evals/v2/fns/requestRuleActivation";import { formatEvaluatorCostCalculation } from "@/src/features/evals/v2/fns/formatEvaluatorCostCalculation";import { compactNumberFormatter, usdFormatter } from "@/src/utils/numbers";@@ -18,6 +19,7 @@ export function EvaluatorSavedCostSummary({isEstimating,onSamplingChange,evaluatorType,+ backfill,}: {estimates: ActivationEstimate[];unavailableEstimateCount: number;@@ -26,6 +28,15 @@ export function EvaluatorSavedCostSummary({isEstimating: boolean;onSamplingChange: ((sampling: number) => void) | null;evaluatorType: EvalTemplateType;+ backfill:+ | { enabled: false }+ | {+ enabled: true;+ matchingObservations: number;+ maxItems: number;+ testRunCostUsd: number | null;+ isEstimating: boolean;+ };}) {const estimate = estimates[0];const sampledObservations = Math.round(matchingObservations * sampling);@@ -35,6 +46,16 @@ export function EvaluatorSavedCostSummary({: estimate? estimate.matchingObservations * sampling * estimate.testRunCostUsd: null;+ const backfillObservationCount = backfill.enabled+ ? Math.min(backfill.matchingObservations, backfill.maxItems)+ : 0;+ const backfillTestRunCostUsd = backfill.enabled+ ? backfill.testRunCostUsd+ : null;+ const backfillEstimatedCostUsd =+ backfillTestRunCostUsd === null+ ? null+ : backfillObservationCount * sampling * backfillTestRunCostUsd;return (<div className="space-y-5">@@ -60,27 +81,29 @@ export function EvaluatorSavedCostSummary({/></section>- <section>- <h3 className="text-sm font-bold">Matches</h3>- {isEstimating ? (- <div className="mt-2 space-y-2">- <Skeleton className="h-5 w-24" />- <Skeleton className="h-3 w-16" />- </div>- ) : (- <>- <p className="mt-1 font-mono text-base font-bold tabular-nums">- {compactNumberFormatter(matchingObservations, 1)} / week- </p>- <p className="text-muted-foreground text-xs tabular-nums">- {compactNumberFormatter(sampledObservations, 1)} sampled- </p>- </>- )}- </section>+ {evaluatorType === EvalTemplateTypeEnum.CODE ? (+ <section>+ <h3 className="text-sm font-bold">Matches</h3>+ {isEstimating ? (+ <div className="mt-2 space-y-2">+ <Skeleton className="h-5 w-24" />+ <Skeleton className="h-3 w-16" />+ </div>+ ) : (+ <>+ <p className="mt-1 font-mono text-base font-bold tabular-nums">+ {compactNumberFormatter(matchingObservations, 1)} / week+ </p>+ <p className="text-muted-foreground text-xs tabular-nums">+ {compactNumberFormatter(sampledObservations, 1)} sampled+ </p>+ </>+ )}+ </section>+ ) : null}{evaluatorType !== EvalTemplateTypeEnum.CODE ? (- <section className="border-t border-dashed pt-4">+ <section className="border-t pt-4">{isEstimating ? (<div className="space-y-2"><Skeleton className="h-6 w-24" />@@ -88,6 +111,9 @@ export function EvaluatorSavedCostSummary({</div>) : (<>+ <p className="text-muted-foreground mb-1 font-mono text-[10px] tracking-wider uppercase">+ Recurring+ </p><div className="flex items-center gap-1.5"><span className="font-mono text-lg font-bold tabular-nums">{estimatedCostUsd === null@@ -95,13 +121,15 @@ export function EvaluatorSavedCostSummary({: `≈ ${usdFormatter(estimatedCostUsd, 2, 2)}`}</span><InfoTooltip label="How estimated LLM costs are calculated">- {formatEvaluatorCostCalculation({- matchingObservations,- sampling,- testRunCostUsd: estimate?.testRunCostUsd ?? null,- estimatedCostUsd,- evaluatorType,- })}+ <EvaluatorCostCalculationTooltipContent+ {...formatEvaluatorCostCalculation({+ matchingObservations,+ sampling,+ testRunCostUsd: estimate?.testRunCostUsd ?? null,+ estimatedCostUsd,+ evaluatorType,+ })}+ /></InfoTooltip></div><p className="text-muted-foreground text-xs">@@ -117,6 +145,45 @@ export function EvaluatorSavedCostSummary({)}</section>) : null}++ {evaluatorType !== EvalTemplateTypeEnum.CODE && backfill.enabled ? (+ <section className="border-t pt-4">+ {backfill.isEstimating ? (+ <div className="space-y-2">+ <Skeleton className="h-6 w-24" />+ <Skeleton className="h-3 w-20" />+ </div>+ ) : (+ <>+ <p className="text-muted-foreground mb-1 font-mono text-[10px] tracking-wider uppercase">+ One-time backfill+ </p>+ <div className="flex items-center gap-1.5">+ <span className="font-mono text-lg font-bold tabular-nums">+ {backfillEstimatedCostUsd !== null+ ? `≈ ${usdFormatter(backfillEstimatedCostUsd, 2, 2)}`+ : "Unavailable"}+ </span>+ <InfoTooltip label="How estimated backfill costs are calculated">+ <EvaluatorCostCalculationTooltipContent+ {...formatEvaluatorCostCalculation({+ matchingObservations: backfillObservationCount,+ sampling,+ testRunCostUsd: backfillTestRunCostUsd,+ estimatedCostUsd: backfillEstimatedCostUsd,+ evaluatorType,+ period: "selection",+ })}+ />+ </InfoTooltip>+ </div>+ <p className="text-muted-foreground text-xs">+ estimated LLM costs, once+ </p>+ </>+ )}+ </section>+ ) : null}</div>);}diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedDialog.clienttest.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedDialog.clienttest.tsxindex ef35c74867..caf33c3935 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedDialog.clienttest.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedDialog.clienttest.tsx@@ -5,6 +5,7 @@ import { EvaluatorSavedDialog } from "./EvaluatorSavedDialog";const renderDialog = () => {const onDismiss = vi.fn();const onSecondaryAction = vi.fn();+ const onModeChange = vi.fn();render(<TooltipProvider>@@ -15,11 +16,12 @@ const renderDialog = () => {"test-filters": null,"different-scope": null,}}+ backfillContent={null}costSummary={null}canSubmitisSubmitting={false}primaryActionLabel="Execute"- onModeChange={vi.fn()}+ onModeChange={onModeChange}onDismiss={onDismiss}onSecondaryAction={onSecondaryAction}onPrimaryAction={vi.fn()}@@ -27,10 +29,22 @@ const renderDialog = () => {</TooltipProvider>,);- return { onDismiss, onSecondaryAction };+ return { onDismiss, onSecondaryAction, onModeChange };};describe("EvaluatorSavedDialog", () => {+ it("selects an inactive mode when its card is clicked", () => {+ const { onModeChange } = renderDialog();++ fireEvent.click(+ screen.getByText(+ "Attach to a rule you already have, or create a new one.",+ ),+ );++ expect(onModeChange).toHaveBeenCalledWith("different-scope");+ });+it("treats the close button as a passive dismissal", () => {const { onDismiss, onSecondaryAction } = renderDialog();diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedDialog.stories.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedDialog.stories.tsxindex 2b2762a881..d829f8ac4c 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedDialog.stories.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedDialog.stories.tsx@@ -6,6 +6,7 @@ import { fn } from "storybook/test";import { EvaluatorSavedCostSummary } from "./EvaluatorSavedCostSummary";import { EvaluatorSavedDialog } from "./EvaluatorSavedDialog";import { EvaluatorSavedRuleFilterPreview } from "./EvaluatorSavedRuleFilterPreview";+import { EvaluatorBackfillSettings } from "../EvaluatorBackfillSettings/EvaluatorBackfillSettings";const meta = preview.meta({ component: EvaluatorSavedDialog });@@ -120,6 +121,25 @@ const unsupportedFilterModeContent = {const sharedArgs = {open: true,modeContentByMode,+ backfillContent: (+ <EvaluatorBackfillSettings+ enabled={false}+ canEnable+ selectedWindow="7-days"+ range={{+ from: new Date("2026-08-28T00:00:00"),+ to: new Date("2026-09-04T23:59:59.999"),+ }}+ maxItems={5_000}+ maxAllowedItems={25_000}+ matchingObservations={4_400}+ isEstimating={false}+ onEnabledChange={fn()}+ onWindowChange={fn()}+ onRangeChange={fn()}+ onMaxItemsChange={fn()}+ />+ ),canSubmit: true,isSubmitting: false,onModeChange: fn(),@@ -149,6 +169,7 @@ export const FromTestFilters = meta.story({sampling={0.4}isEstimating={false}evaluatorType="LLM_AS_JUDGE"+ backfill={{ enabled: false }}onSamplingChange={fn()}/>),@@ -156,6 +177,110 @@ export const FromTestFilters = meta.story({},});+export const CodeEvaluator = meta.story({+ args: {+ ...sharedArgs,+ mode: "test-filters",+ costSummary: (+ <EvaluatorSavedCostSummary+ estimates={[]}+ unavailableEstimateCount={0}+ matchingObservations={80}+ sampling={1}+ isEstimating={false}+ evaluatorType="CODE"+ backfill={{ enabled: false }}+ onSamplingChange={fn()}+ />+ ),+ primaryActionLabel: "Execute",+ },+});++const backfillCostSummary = (+ <EvaluatorSavedCostSummary+ estimates={[+ {+ evaluatorId: "evaluator-1",+ evaluatorName: "Conciseness",+ matchingObservations: 4_400,+ sampling: 1,+ testRunCostUsd: 0.0017,+ estimatedCostUsd: 7.48,+ },+ ]}+ unavailableEstimateCount={0}+ matchingObservations={4_400}+ sampling={1}+ isEstimating={false}+ evaluatorType="LLM_AS_JUDGE"+ backfill={{+ enabled: true,+ matchingObservations: 4_400,+ maxItems: 5_000,+ testRunCostUsd: 0.0017,+ isEstimating: false,+ }}+ onSamplingChange={fn()}+ />+);++export const WithBackfill = meta.story({+ args: {+ ...sharedArgs,+ mode: "test-filters",+ backfillContent: (+ <EvaluatorBackfillSettings+ enabled+ canEnable+ selectedWindow="7-days"+ range={{+ from: new Date("2026-08-28T00:00:00"),+ to: new Date("2026-09-04T23:59:59.999"),+ }}+ maxItems={5_000}+ maxAllowedItems={25_000}+ matchingObservations={4_400}+ isEstimating={false}+ onEnabledChange={fn()}+ onWindowChange={fn()}+ onRangeChange={fn()}+ onMaxItemsChange={fn()}+ />+ ),+ costSummary: backfillCostSummary,+ primaryActionLabel: "Execute",+ },+});++export const WithCustomBackfill = meta.story({+ args: {+ ...sharedArgs,+ mode: "test-filters",+ backfillContent: (+ <EvaluatorBackfillSettings+ enabled+ canEnable+ selectedWindow="custom"+ range={{+ from: new Date("2026-08-01T00:00:00"),+ to: new Date("2026-09-04T23:59:59.999"),+ }}+ maxItems={5_000}+ maxAllowedItems={25_000}+ matchingObservations={21_420}+ isEstimating={false}+ onEnabledChange={fn()}+ onWindowChange={fn()}+ onRangeChange={fn()}+ onMaxItemsChange={fn()}+ />+ ),+ costSummary: backfillCostSummary,+ primaryActionLabel: "Execute",+ },+});+export const UnsupportedFilters = meta.story({args: {...sharedArgs,@@ -178,6 +303,7 @@ export const UnsupportedFilters = meta.story({sampling={0.4}isEstimating={false}evaluatorType="LLM_AS_JUDGE"+ backfill={{ enabled: false }}onSamplingChange={fn()}/>),@@ -206,6 +332,7 @@ export const ExistingRule = meta.story({sampling={0.25}isEstimating={false}evaluatorType="LLM_AS_JUDGE"+ backfill={{ enabled: false }}onSamplingChange={null}/>),diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedDialog.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedDialog.tsxindex a5fdd50f16..91994f408c 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedDialog.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedDialog.tsx@@ -41,6 +41,7 @@ export function EvaluatorSavedDialog({open,mode,modeContentByMode,+ backfillContent,costSummary,canSubmit,isSubmitting,@@ -49,11 +50,13 @@ export function EvaluatorSavedDialog({onDismiss,onSecondaryAction,onPrimaryAction,+ onOpenAutoFocus,onCloseAnimationEnd,}: {open: boolean;mode: EvaluatorSavedMode;modeContentByMode: Record<EvaluatorSavedMode, ReactNode>;+ backfillContent: ReactNode;costSummary: ReactNode;canSubmit: boolean;isSubmitting: boolean;@@ -62,6 +65,7 @@ export function EvaluatorSavedDialog({onDismiss: () => void;onSecondaryAction: () => void;onPrimaryAction: () => void;+ onOpenAutoFocus?: () => void;onCloseAnimationEnd?: () => void;}) {return (@@ -74,16 +78,17 @@ export function EvaluatorSavedDialog({<DialogContentclassName="sm:max-w-4xl"closeOnInteractionOutside+ onOpenAutoFocus={onOpenAutoFocus}onCloseAutoFocus={onCloseAnimationEnd}><DialogHeader className="[&>div]:items-start [&>div>button]:-mt-1"><DialogTitle>Evaluator saved</DialogTitle><DialogDescription>- Would you like to run this evaluator on incoming observations?+ Choose which observations this evaluator should score.</DialogDescription></DialogHeader><DialogBody className="gap-0 p-0">- <div className="grid h-[22rem] grid-cols-[minmax(0,1fr)_15rem] overflow-hidden">+ <div className="grid h-[30rem] grid-cols-[minmax(0,1fr)_15rem] overflow-hidden"><div className="min-w-0 overflow-y-auto px-6 py-5 [scrollbar-gutter:stable]"><h3 className="mb-2 text-sm font-bold">Set up rule to run on incoming observations@@ -103,11 +108,23 @@ export function EvaluatorSavedDialog({<Collapsible key={option.value} open={selected} asChild><divclassName={cn(- "rounded-md border p-3 transition-colors duration-200",+ "min-w-0 rounded-md border p-3 transition-colors duration-200",selected? "border-foreground bg-background"- : "bg-muted/30 hover:bg-muted/50",+ : "bg-muted/30 hover:bg-muted/50 cursor-pointer",)}+ onClick={(event) => {+ if (selected) return;+ const target = event.target as HTMLElement;+ if (+ target.closest(+ "button, input, label, a, [role='radio']",+ )+ ) {+ return;+ }+ onModeChange(option.value);+ }}><div className="flex items-start gap-2"><div className="mt-0.5">@@ -132,11 +149,12 @@ export function EvaluatorSavedDialog({<CollapsibleContentid={contentId}aria-labelledby={id}- className={styles.collapsibleContent}+ className={cn(styles.collapsibleContent, "space-y-3")}><div className="mt-3 ml-6 min-w-0 pr-1">{modeContentByMode[option.value]}</div>+ {backfillContent}</CollapsibleContent></div></Collapsible>diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsxindex 59cafeb9b7..359a2c54a0 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsx@@ -6,7 +6,7 @@ import {type ObservationVariableMapping,} from "@langfuse/shared";import { ChevronDown } from "lucide-react";-import { useCallback, useEffect, useMemo, useRef, useState } from "react";+import { useCallback, useMemo, useRef, useState } from "react";import { Button } from "@/src/components/ui/button";import { PopoverTrigger } from "@/src/components/ui/popover";import { selectTriggerClassName } from "@/src/components/ui/select";@@ -26,10 +26,22 @@ import { cn } from "@/src/utils/tailwind";import { classifySampleFiltersForRule } from "@/src/features/evals/v2/fns/rules/classifySampleFiltersForRule";import { getFilterAnalyticsProperties } from "@/src/features/evals/v2/fns/getFilterAnalyticsProperties";import { EvaluatorSavedRuleFilterPreview } from "@/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedRuleFilterPreview";+import { EvaluatorBackfillSettings } from "@/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/EvaluatorBackfillSettings";+import { useEvaluatorSavedBackfill } from "@/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/hooks/useEvaluatorSavedBackfill";type Rule = RouterOutputs["evalsV2"]["rules"]["list"]["rules"][number];type DialogPhase = "saved" | "closing-saved" | "create-rule" | "closed";+type EvaluatorSavedDialogEvaluator = {+ id: string;+ name: string;+ type: EvalTemplateType;+ defaultVariableMapping: ObservationVariableMapping[];+ sampleFilter: FilterState;+ hasCompletedTestCall?: boolean;+ testRunCostUsd?: number | null;+};+export function EvaluatorSavedDialogContainer({projectId,evaluator,@@ -37,15 +49,7 @@ export function EvaluatorSavedDialogContainer({onFinish,}: {projectId: string;- evaluator: {- id: string;- name: string;- type: EvalTemplateType;- defaultVariableMapping: ObservationVariableMapping[];- sampleFilter: FilterState;- hasCompletedTestCall?: boolean;- testRunCostUsd?: number | null;- };+ evaluator: EvaluatorSavedDialogEvaluator;onDismiss: () => Promise<void>;onFinish: () => Promise<void>;}) {@@ -63,13 +67,21 @@ export function EvaluatorSavedDialogContainer({>(undefined);const [isEstimating, setIsEstimating] = useState(false);const [testFilterSampling, setTestFilterSampling] = useState(1);+ const [isCompleting, setIsCompleting] = useState(false);const estimateRequestId = useRef(0);- // Strict Mode replays mount effects; this mutation must run once per dialog.const initialEstimateRequested = useRef(false);const createRuleHandoffPending = useRef(false);const hasRequestedMissingCostTest = useRef(evaluator.hasCompletedTestCall ?? false,);+ const claimMissingCostTest = useCallback(() => {+ if (hasRequestedMissingCostTest.current) return false;+ hasRequestedMissingCostTest.current = true;+ return true;+ }, []);+ const resetMissingCostTest = useCallback(() => {+ hasRequestedMissingCostTest.current = false;+ }, []);const missingCostTestRequest = useRef<Promise<void> | null>(null);const activeRules = api.evalsV2.rules.list.useQuery({@@ -91,6 +103,17 @@ export function EvaluatorSavedDialogContainer({},{ enabled: dialogPhase === "saved" },);+ const historicEvaluationLimit = api.evals.globalJobConfigs.useQuery({+ projectId,+ });+ const backfill = useEvaluatorSavedBackfill({+ projectId,+ evaluatorId: evaluator.id,+ knownTestRunCostUsd: evaluator.testRunCostUsd ?? undefined,+ historicEvaluationLimit: historicEvaluationLimit.data,+ claimMissingCostTest,+ resetMissingCostTest,+ });const availableRules = useMemo(() =>[@@ -137,17 +160,35 @@ export function EvaluatorSavedDialogContainer({};const attachToRule = async (rule: Rule) => {- await attach.mutateAsync({+ const currentAssignments =+ await utils.client.evalsV2.rules.listRulesForEvaluator.query({+ projectId,+ evaluatorId: evaluator.id,+ });+ const isAlreadyAttached = currentAssignments.some(+ (assignment) => assignment.evaluationRule.id === rule.id,+ );+ if (!isAlreadyAttached) {+ await attach.mutateAsync({+ projectId,+ ruleId: rule.id,+ evaluatorId: evaluator.id,+ variableMapping: null,+ enableRule: !rule.enabled,+ });+ capture("evaluation_rules:attach_evaluator", {+ evaluatorCount: 1,+ source: "evaluator_create",+ });+ }+ const currentRule = await utils.client.evalsV2.rules.get.query({projectId,ruleId: rule.id,- evaluatorId: evaluator.id,- variableMapping: null,- enableRule: !rule.enabled,- });- capture("evaluation_rules:attach_evaluator", {- evaluatorCount: 1,- source: "evaluator_create",});+ await backfill.schedule(+ { filter: currentRule.filter, sampling: currentRule.sampling },+ backfill.executionRange,+ );await invalidateRuleQueries();await finish();};@@ -174,6 +215,10 @@ export function EvaluatorSavedDialogContainer({source: "evaluator_create_test_filters",});}+ await backfill.schedule(+ { filter: supportedRuleFilters, sampling },+ backfill.executionRange,+ );await invalidateRuleQueries();await finish();};@@ -192,10 +237,9 @@ export function EvaluatorSavedDialogContainer({}if (estimateRequestId.current !== requestId) return;- const shouldRunMissingTest = !hasRequestedMissingCostTest.current;+ const shouldRunMissingTest = claimMissingCostTest();let finishMissingCostTestRequest: (() => void) | undefined;if (shouldRunMissingTest) {- hasRequestedMissingCostTest.current = true;missingCostTestRequest.current = new Promise<void>((resolve) => {finishMissingCostTestRequest = resolve;});@@ -231,42 +275,32 @@ export function EvaluatorSavedDialogContainer({if (shouldRunMissingTest) missingCostTestRequest.current = null;}if (estimateRequestId.current !== requestId) return;- if (result?.matchingObservations === 0) {- hasRequestedMissingCostTest.current = false;+ if (shouldRunMissingTest && result?.matchingObservations === 0) {+ resetMissingCostTest();}} finally {if (estimateRequestId.current === requestId) setIsEstimating(false);}},[+ claimMissingCostTest,evaluator.id,evaluator.name,evaluator.testRunCostUsd,requestActivation,+ resetMissingCostTest,setActivationOpen,],);- useEffect(() => {- if (- dialogPhase !== "saved" ||- mode !== "test-filters" ||- initialEstimateRequested.current- ) {- return;- }+ const requestInitialEstimate = () => {+ if (initialEstimateRequested.current) return;initialEstimateRequested.current = true;requestEstimate({filter: supportedRuleFilters,sampling: testFilterSampling,}).catch(() => undefined);- }, [- dialogPhase,- mode,- requestEstimate,- supportedRuleFilters,- testFilterSampling,- ]);+ };const openCreateRule = () => {createRuleHandoffPending.current = true;@@ -287,68 +321,115 @@ export function EvaluatorSavedDialogContainer({requestEstimate({ filter: rule.filter, sampling: rule.sampling }).catch(() => undefined,);+ if (backfill.enabled) {+ backfill+ .requestEstimate({ filter: rule.filter, sampling: rule.sampling })+ .catch(() => undefined);+ }},- [requestEstimate, setActivationSampling],+ [backfill, requestEstimate, setActivationSampling],);const selectNewRule = () => {estimateRequestId.current += 1;setActivationOpen(false);- setSelectedRuleId(null);setIsEstimating(false);+ setSelectedRuleId(null);+ backfill.clearScope();};- useEffect(() => {- if (- mode !== "different-scope" ||- selectedRuleId !== undefined ||- rulesPending- ) {- return;- }-- const mostUsedRule = availableRules[0];- if (mostUsedRule) {- selectExistingRule(mostUsedRule);- } else {- setSelectedRuleId(null);- }- }, [availableRules, mode, rulesPending, selectExistingRule, selectedRuleId]);-const handleModeChange = (nextMode: EvaluatorSavedMode) => {- estimateRequestId.current += 1;- if (nextMode !== "test-filters") initialEstimateRequested.current = false;+ const modeChangeRequestId = ++estimateRequestId.current;+ setActivationOpen(false);+ setIsEstimating(false);setMode(nextMode);- const mostUsedRule = availableRules[0];- if (nextMode === "different-scope" && mostUsedRule) {- selectExistingRule(mostUsedRule);++ if (nextMode === "different-scope") {+ const selectMostUsedRule = (rules: Rule[]) => {+ if (estimateRequestId.current !== modeChangeRequestId) return;+ const mostUsedRule = rules[0];+ if (mostUsedRule) {+ selectExistingRule(mostUsedRule);+ return;+ }+ setSelectedRuleId(null);+ backfill.clearScope();+ };++ if (rulesPending) {+ setSelectedRuleId(undefined);+ Promise.all([activeRules.refetch(), inactiveRules.refetch()])+ .then(([activeResult, inactiveResult]) =>+ selectMostUsedRule(+ [+ ...(activeResult.data?.rules ?? []),+ ...(inactiveResult.data?.rules ?? []),+ ].sort(+ (left, right) =>+ right.assignments.length - left.assignments.length,+ ),+ ),+ )+ .catch(() => selectMostUsedRule([]));+ return;+ }++ selectMostUsedRule(availableRules);return;}- setSelectedRuleId(- nextMode === "different-scope" && !rulesPending ? null : undefined,- );- setIsEstimating(false);- if (nextMode === "test-filters") {- setActivationSampling(testFilterSampling);++ setSelectedRuleId(undefined);+ setActivationSampling(testFilterSampling);+ requestEstimate({+ filter: supportedRuleFilters,+ sampling: testFilterSampling,+ }).catch(() => undefined);+ if (backfill.enabled) {+ backfill+ .requestEstimate({+ filter: supportedRuleFilters,+ sampling: testFilterSampling,+ })+ .catch(() => undefined);}};+ const currentBackfillScope =+ mode === "test-filters"+ ? { filter: supportedRuleFilters, sampling: testFilterSampling }+ : selectedRule+ ? { filter: selectedRule.filter, sampling: selectedRule.sampling }+ : null;+const handlePrimaryAction = () => {if (mode === "test-filters") {capture("evaluators:saved_dialog_submit", {action: "test_filters",+ hasBackfill: backfill.enabled,+ backfillWindow: backfill.enabled ? backfill.window : undefined,+ backfillMaxItems: backfill.enabled+ ? backfill.effectiveMaxItems+ : undefined,});- resolveFromTestFilters().catch(() => undefined);+ setIsCompleting(true);+ resolveFromTestFilters().catch(() => setIsCompleting(false));return;}if (selectedRule) {capture("evaluators:saved_dialog_submit", {action: "existing_rule",+ hasBackfill: backfill.enabled,+ backfillWindow: backfill.enabled ? backfill.window : undefined,+ backfillMaxItems: backfill.enabled+ ? backfill.effectiveMaxItems+ : undefined,});- attachToRule(selectedRule).catch(() => undefined);+ setIsCompleting(true);+ attachToRule(selectedRule).catch(() => setIsCompleting(false));} else {capture("evaluators:saved_dialog_submit", {action: "new_rule",+ hasBackfill: false,});openCreateRule();}@@ -423,6 +504,28 @@ export function EvaluatorSavedDialogContainer({};const hasConfiguredScope = mode === "test-filters" || Boolean(selectedRule);+ const backfillContent = (+ <EvaluatorBackfillSettings+ enabled={backfill.enabled}+ canEnable={hasConfiguredScope}+ selectedWindow={backfill.window}+ range={backfill.range}+ maxItems={backfill.effectiveMaxItems}+ maxAllowedItems={backfill.allowedItems}+ matchingObservations={backfill.matchingObservations}+ isEstimating={backfill.isEstimating}+ onEnabledChange={(enabled) =>+ backfill.setEnabled(enabled, currentBackfillScope)+ }+ onWindowChange={(window) =>+ backfill.updateWindow(window, currentBackfillScope)+ }+ onRangeChange={(range) =>+ backfill.updateRange(range, currentBackfillScope)+ }+ onMaxItemsChange={backfill.setMaxItems}+ />+ );const costSummary = hasConfiguredScope ? (<EvaluatorSavedCostSummaryestimates={activation.estimate.estimates}@@ -435,6 +538,17 @@ export function EvaluatorSavedDialogContainer({}isEstimating={isEstimating}evaluatorType={evaluator.type}+ backfill={+ backfill.enabled+ ? {+ enabled: true,+ matchingObservations: backfill.matchingObservations,+ maxItems: backfill.effectiveMaxItems,+ testRunCostUsd: backfill.testRunCostUsd,+ isEstimating: backfill.isEstimating,+ }+ : { enabled: false }+ }onSamplingChange={mode === "test-filters"? (sampling) => {@@ -458,13 +572,20 @@ export function EvaluatorSavedDialogContainer({open={dialogPhase === "saved"}mode={mode}modeContentByMode={modeContentByMode}+ backfillContent={backfillContent}costSummary={costSummary}canSubmit={!isEstimating &&+ !isCompleting &&+ (!backfill.enabled || !backfill.isEstimating) &&+ (!backfill.enabled || historicEvaluationLimit.isSuccess) &&(mode === "test-filters" || selectedRuleId !== undefined)}isSubmitting={- attach.isPending || createOrAttachFromEvaluatorFilters.isPending+ attach.isPending ||+ createOrAttachFromEvaluatorFilters.isPending ||+ backfill.isScheduling ||+ isCompleting}primaryActionLabel={mode === "test-filters"@@ -474,6 +595,7 @@ export function EvaluatorSavedDialogContainer({: "Open rule editor"}onModeChange={handleModeChange}+ onOpenAutoFocus={requestInitialEstimate}onDismiss={() => {createRuleHandoffPending.current = false;setDialogPhase("closed");diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/hooks/useEvaluatorSavedBackfill.ts b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/hooks/useEvaluatorSavedBackfill.tsnew file mode 100644index 0000000000..3d1a705540--- /dev/null+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/hooks/useEvaluatorSavedBackfill.ts@@ -0,0 +1,267 @@+import { BatchEvalSourceTable, type FilterState } from "@langfuse/shared";+import { endOfDay, startOfDay, subDays, subHours, subMonths } from "date-fns";+import { useCallback, useRef, useState } from "react";+import type {+ EvaluatorBackfillRange,+ EvaluatorBackfillWindow,+} from "@/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/EvaluatorBackfillSettings";+import {+ DEFAULT_EVALUATOR_BACKFILL_ITEMS,+ MAX_EVALUATOR_BACKFILL_ITEMS,+} from "@/src/features/evals/v2/constants/evaluatorBackfill";+import { api } from "@/src/utils/api";+import { trpcErrorToast } from "@/src/utils/trpcErrorToast";++type EvaluatorBackfillScope = {+ filter: FilterState;+ sampling: number;+};++function getBackfillRange(+ window: Exclude<EvaluatorBackfillWindow, "custom">,+ now = new Date(),+): EvaluatorBackfillRange {+ switch (window) {+ case "24-hours":+ return { from: subHours(now, 24), to: now };+ case "7-days":+ return { from: subDays(now, 7), to: now };+ case "30-days":+ return { from: subDays(now, 30), to: now };+ case "90-days":+ return { from: subDays(now, 90), to: now };+ }+}++function normalizeBackfillRange(+ nextRange: EvaluatorBackfillRange,+ previousFrom: Date,+ now = new Date(),+): EvaluatorBackfillRange {+ const earliestAllowedStart = startOfDay(subMonths(now, 6));+ const latestAllowedEnd = endOfDay(now);+ const clampDate = (date: Date) =>+ new Date(+ Math.min(+ Math.max(date.getTime(), earliestAllowedStart.getTime()),+ latestAllowedEnd.getTime(),+ ),+ );+ let from = clampDate(nextRange.from);+ let to = clampDate(nextRange.to);++ if (from > to) {+ if (nextRange.from.getTime() !== previousFrom.getTime()) {+ to = clampDate(endOfDay(from));+ } else {+ from = clampDate(startOfDay(to));+ }+ }++ return { from, to };+}++export function useEvaluatorSavedBackfill({+ projectId,+ evaluatorId,+ knownTestRunCostUsd,+ historicEvaluationLimit,+ claimMissingCostTest,+ resetMissingCostTest,+}: {+ projectId: string;+ evaluatorId: string;+ knownTestRunCostUsd?: number;+ historicEvaluationLimit?: number;+ claimMissingCostTest: () => boolean;+ resetMissingCostTest: () => void;+}) {+ const utils = api.useUtils();+ const runEvaluation = api.batchAction.runEvaluation.create.useMutation({+ onError: trpcErrorToast,+ });+ const [enabled, setEnabledState] = useState(false);+ const [window, setWindow] = useState<EvaluatorBackfillWindow>("7-days");+ const [range, setRange] = useState<EvaluatorBackfillRange>(() =>+ getBackfillRange("7-days"),+ );+ const [maxItems, setMaxItems] = useState(DEFAULT_EVALUATOR_BACKFILL_ITEMS);+ const [matchingObservations, setMatchingObservations] = useState(0);+ const [testRunCostUsd, setTestRunCostUsd] = useState<number | null>(null);+ const [isEstimating, setIsEstimating] = useState(false);+ const estimateRequestId = useRef(0);+ const hasScheduled = useRef(false);++ const requestEstimate = useCallback(+ async (scope: EvaluatorBackfillScope, estimateRange = range) => {+ const requestId = ++estimateRequestId.current;+ setIsEstimating(true);+ try {+ const shouldRunMissingTest = claimMissingCostTest();+ const result =+ await utils.client.evalsV2.activationCostEstimates.mutate({+ projectId,+ evaluatorIds: [evaluatorId],+ filter: scope.filter,+ sampling: scope.sampling,+ shouldRunMissingTest,+ timeRange: estimateRange,+ ...(knownTestRunCostUsd !== undefined+ ? { knownTestRunCostUsd }+ : {}),+ });+ if (estimateRequestId.current !== requestId) return;+ const estimate = result.find(+ ({ evaluatorId: resultEvaluatorId }) =>+ resultEvaluatorId === evaluatorId,+ );+ if (shouldRunMissingTest && estimate?.matchingObservations === 0) {+ resetMissingCostTest();+ }+ setMatchingObservations(estimate?.matchingObservations ?? 0);+ setTestRunCostUsd(estimate?.testRunCostUsd ?? null);+ } catch (error) {+ if (estimateRequestId.current === requestId) {+ setMatchingObservations(0);+ setTestRunCostUsd(null);+ trpcErrorToast(error);+ }+ } finally {+ if (estimateRequestId.current === requestId) {+ setIsEstimating(false);+ }+ }+ },+ [+ claimMissingCostTest,+ evaluatorId,+ knownTestRunCostUsd,+ projectId,+ range,+ resetMissingCostTest,+ utils.client,+ ],+ );++ const clearScope = useCallback(() => {+ estimateRequestId.current += 1;+ setIsEstimating(false);+ setEnabledState(false);+ setMatchingObservations(0);+ setTestRunCostUsd(null);+ }, []);++ const setEnabled = useCallback(+ (nextEnabled: boolean, scope: EvaluatorBackfillScope | null) => {+ setEnabledState(nextEnabled);+ if (!nextEnabled) {+ estimateRequestId.current += 1;+ setIsEstimating(false);+ return;+ }++ const defaultWindow: EvaluatorBackfillWindow = "7-days";+ const defaultRange = getBackfillRange(defaultWindow);+ setWindow(defaultWindow);+ setRange(defaultRange);+ if (scope) {+ requestEstimate(scope, defaultRange).catch(() => undefined);+ }+ },+ [requestEstimate],+ );++ const updateRange = useCallback(+ (+ nextRange: EvaluatorBackfillRange,+ scope: EvaluatorBackfillScope | null,+ ) => {+ const clampedRange = normalizeBackfillRange(nextRange, range.from);+ setRange(clampedRange);+ if (scope) {+ requestEstimate(scope, clampedRange).catch(() => undefined);+ }+ },+ [range.from, requestEstimate],+ );++ const updateWindow = useCallback(+ (+ nextWindow: EvaluatorBackfillWindow,+ scope: EvaluatorBackfillScope | null,+ ) => {+ setWindow(nextWindow);+ if (nextWindow !== "custom") {+ updateRange(getBackfillRange(nextWindow), scope);+ }+ },+ [updateRange],+ );++ const allowedItems = Math.min(+ historicEvaluationLimit ?? MAX_EVALUATOR_BACKFILL_ITEMS,+ MAX_EVALUATOR_BACKFILL_ITEMS,+ );+ const effectiveMaxItems = Math.min(maxItems, allowedItems);+ const executionRange = window === "custom" ? range : getBackfillRange(window);++ const schedule = async (+ scope: EvaluatorBackfillScope,+ executionRange: EvaluatorBackfillRange,+ ) => {+ if (!enabled || hasScheduled.current) return;+ const normalizedExecutionRange = normalizeBackfillRange(+ executionRange,+ range.from,+ );+ await runEvaluation.mutateAsync({+ projectId,+ query: {+ filter: [+ ...scope.filter,+ {+ column: "startTime",+ type: "datetime",+ operator: ">=",+ value: normalizedExecutionRange.from,+ },+ {+ column: "startTime",+ type: "datetime",+ operator: "<=",+ value: normalizedExecutionRange.to,+ },+ ],+ orderBy: { column: "startTime", order: "DESC" },+ useEventsTable: true,+ },+ evaluatorIds: [evaluatorId],+ evaluatorMappings: [{ evaluatorId, variableMapping: null }],+ evalVersion: "v2",+ sourceTable: BatchEvalSourceTable.EVENTS,+ sampling: scope.sampling,+ rowLimit: effectiveMaxItems,+ });+ hasScheduled.current = true;+ };++ return {+ enabled,+ window,+ range,+ effectiveMaxItems,+ allowedItems,+ matchingObservations,+ testRunCostUsd,+ isEstimating,+ isScheduling: runEvaluation.isPending,+ executionRange,+ requestEstimate,+ clearScope,+ setEnabled,+ updateRange,+ updateWindow,+ setMaxItems,+ schedule,+ };+}diff --git a/web/src/features/evals/v2/components/Rules/RuleSetup/components/RuleEvaluatorCostEstimate.tsx b/web/src/features/evals/v2/components/Rules/RuleSetup/components/RuleEvaluatorCostEstimate.tsxindex 734581266a..f5e992529d 100644--- a/web/src/features/evals/v2/components/Rules/RuleSetup/components/RuleEvaluatorCostEstimate.tsx+++ b/web/src/features/evals/v2/components/Rules/RuleSetup/components/RuleEvaluatorCostEstimate.tsx@@ -3,6 +3,7 @@ import {TooltipContent,TooltipTrigger,} from "@/src/components/ui/tooltip";+import { EvaluatorCostCalculationTooltipContent } from "@/src/features/evals/v2/components/EvaluatorCostCalculationTooltipContent/EvaluatorCostCalculationTooltipContent";import { formatEvaluatorCostCalculation } from "@/src/features/evals/v2/fns/formatEvaluatorCostCalculation";import type { RuleCostEstimate } from "@/src/features/evals/v2/hooks/useRuleCostEstimate";import { usdFormatter } from "@/src/utils/numbers";@@ -24,7 +25,9 @@ export function RuleEvaluatorCostEstimate({</span></TooltipTrigger><TooltipContent className="max-w-72">- {formatEvaluatorCostCalculation(estimate)}+ <EvaluatorCostCalculationTooltipContent+ {...formatEvaluatorCostCalculation(estimate)}+ /></TooltipContent></Tooltip>);diff --git a/web/src/features/evals/v2/constants/evaluatorBackfill.ts b/web/src/features/evals/v2/constants/evaluatorBackfill.tsnew file mode 100644index 0000000000..8107d384de--- /dev/null+++ b/web/src/features/evals/v2/constants/evaluatorBackfill.ts@@ -0,0 +1,2 @@+export const MAX_EVALUATOR_BACKFILL_ITEMS = 25_000;+export const DEFAULT_EVALUATOR_BACKFILL_ITEMS = 5_000;diff --git a/web/src/features/evals/v2/fns/formatEvaluatorCostCalculation.clienttest.ts b/web/src/features/evals/v2/fns/formatEvaluatorCostCalculation.clienttest.tsindex 8d39a5b9eb..365799d888 100644--- a/web/src/features/evals/v2/fns/formatEvaluatorCostCalculation.clienttest.ts+++ b/web/src/features/evals/v2/fns/formatEvaluatorCostCalculation.clienttest.ts@@ -13,9 +13,12 @@ describe("formatEvaluatorCostCalculation", () => {estimatedCostUsd: 0.736,evaluatorType: EvalTemplateTypeEnum.LLM_AS_JUDGE,}),- ).toBe(- "1,840 matching observations × 40% sampling × $0.001 per evaluation = ≈ $0.74 / week. Expected cost on your linked API key (not Langfuse). Matching observations are based on the last 7 days. Per-evaluation cost uses the latest cost-bearing evaluator trace from that period, or a fallback test call.",- );+ ).toEqual({+ calculation:+ "1,840 matching observations × 40% sampling × $0.001 per evaluation ≈ $0.74 / week",+ explanation:+ "Expected cost on your linked API key (not Langfuse). Matching observations are based on the last 7 days. Per-evaluation cost uses the latest cost-bearing evaluator trace from that period, or a fallback test call.",+ });});it("explains when neither a recent trace nor fallback test provides a cost", () => {@@ -27,9 +30,11 @@ describe("formatEvaluatorCostCalculation", () => {estimatedCostUsd: null,evaluatorType: EvalTemplateTypeEnum.LLM_AS_JUDGE,}),- ).toBe(- "No cost-bearing evaluator trace was available from the last 7 days, and the fallback test call did not return a usable model cost. Expected cost would be charged to your linked API key, not Langfuse.",- );+ ).toEqual({+ calculation: null,+ explanation:+ "No cost-bearing evaluator trace was available from the last 7 days, and the fallback test call did not return a usable model cost. Expected cost would be charged to your linked API key, not Langfuse.",+ });});it("shows zero cost when no observations match", () => {@@ -41,9 +46,11 @@ describe("formatEvaluatorCostCalculation", () => {estimatedCostUsd: 0,evaluatorType: EvalTemplateTypeEnum.LLM_AS_JUDGE,}),- ).toBe(- "0 matching observations × 100% sampling = ≈ $0.00 / week. Expected cost on your linked API key (not Langfuse). Matching observations are based on the last 7 days.",- );+ ).toEqual({+ calculation: "0 matching observations × 100% sampling ≈ $0.00 / week",+ explanation:+ "Expected cost on your linked API key (not Langfuse). Matching observations are based on the last 7 days.",+ });});it("explains why code evaluators have no model-provider cost", () => {@@ -55,9 +62,11 @@ describe("formatEvaluatorCostCalculation", () => {estimatedCostUsd: 0,evaluatorType: EvalTemplateTypeEnum.CODE,}),- ).toBe(- "Code evaluators do not call an LLM, so they do not incur model-provider / LLM costs.",- );+ ).toEqual({+ calculation: null,+ explanation:+ "Code evaluators do not call an LLM, so they do not incur model-provider / LLM costs.",+ });});it("does not claim a 7-day window when a selection has no priced test", () => {@@ -70,9 +79,11 @@ describe("formatEvaluatorCostCalculation", () => {evaluatorType: EvalTemplateTypeEnum.LLM_AS_JUDGE,period: "selection",}),- ).toBe(- "No cost-bearing evaluator trace was available, and the fallback test call did not return a usable model cost. Expected cost would be charged to your linked API key, not Langfuse.",- );+ ).toEqual({+ calculation: null,+ explanation:+ "No cost-bearing evaluator trace was available, and the fallback test call did not return a usable model cost. Expected cost would be charged to your linked API key, not Langfuse.",+ });});it("describes a one-shot selection instead of a weekly total", () => {@@ -85,8 +96,11 @@ describe("formatEvaluatorCostCalculation", () => {evaluatorType: EvalTemplateTypeEnum.LLM_AS_JUDGE,period: "selection",}),- ).toBe(- "12 observations × 100% sampling × $0.01 per evaluation = ≈ $0.12 for this run. Expected cost on your linked API key (not Langfuse). Per-evaluation cost uses the latest cost-bearing evaluator trace from that period, or a fallback test call.",- );+ ).toEqual({+ calculation:+ "12 observations × 100% sampling × $0.01 per evaluation ≈ $0.12 for this run",+ explanation:+ "Expected cost on your linked API key (not Langfuse). Per-evaluation cost uses the latest cost-bearing evaluator trace from that period, or a fallback test call.",+ });});});diff --git a/web/src/features/evals/v2/fns/formatEvaluatorCostCalculation.ts b/web/src/features/evals/v2/fns/formatEvaluatorCostCalculation.tsindex 6a4f8faa19..ba528ee951 100644--- a/web/src/features/evals/v2/fns/formatEvaluatorCostCalculation.ts+++ b/web/src/features/evals/v2/fns/formatEvaluatorCostCalculation.ts@@ -17,7 +17,11 @@ export function formatEvaluatorCostCalculation({period?: "week" | "selection";}) {if (evaluatorType === EvalTemplateTypeEnum.CODE) {- return "Code evaluators do not call an LLM, so they do not incur model-provider / LLM costs.";+ return {+ calculation: null,+ explanation:+ "Code evaluators do not call an LLM, so they do not incur model-provider / LLM costs.",+ };}const isSelection = period === "selection";@@ -31,12 +35,21 @@ export function formatEvaluatorCostCalculation({: "Expected cost on your linked API key (not Langfuse). Matching observations are based on the last 7 days.";if (matchingObservations === 0) {- return `${observations} × ${samplingRate} = ≈ ${usdFormatter(0, 2, 2)} ${periodLabel}. ${scope}`;+ return {+ calculation: `${observations} × ${samplingRate} ≈ ${usdFormatter(0, 2, 2)} ${periodLabel}`,+ explanation: scope,+ };}if (testRunCostUsd === null || estimatedCostUsd === null) {const window = isSelection ? "" : " from the last 7 days";- return `No cost-bearing evaluator trace was available${window}, and the fallback test call did not return a usable model cost. Expected cost would be charged to your linked API key, not Langfuse.`;+ return {+ calculation: null,+ explanation: `No cost-bearing evaluator trace was available${window}, and the fallback test call did not return a usable model cost. Expected cost would be charged to your linked API key, not Langfuse.`,+ };}- return `${observations} × ${samplingRate} × ${usdFormatter(testRunCostUsd, 2, 6)} per evaluation = ≈ ${usdFormatter(estimatedCostUsd, 2, 2)} ${periodLabel}. ${scope} Per-evaluation cost uses the latest cost-bearing evaluator trace from that period, or a fallback test call.`;+ return {+ calculation: `${observations} × ${samplingRate} × ${usdFormatter(testRunCostUsd, 2, 6)} per evaluation ≈ ${usdFormatter(estimatedCostUsd, 2, 2)} ${periodLabel}`,+ explanation: `${scope} Per-evaluation cost uses the latest cost-bearing evaluator trace from that period, or a fallback test call.`,+ };}diff --git a/web/src/features/evals/v2/server/evaluators/activationCostService.servertest.ts b/web/src/features/evals/v2/server/evaluators/activationCostService.servertest.tsindex 7bac13d266..94a729bad7 100644--- a/web/src/features/evals/v2/server/evaluators/activationCostService.servertest.ts+++ b/web/src/features/evals/v2/server/evaluators/activationCostService.servertest.ts@@ -29,6 +29,7 @@ vi.mock("./testEvaluator", () => ({}));import { getActivationCostEstimates } from "./activationCostService";+import { ActivationCostEstimatesSchema } from "./evaluatorTypes";const evaluator = {id: "evaluator-id",@@ -78,6 +79,21 @@ describe("getActivationCostEstimates", () => {vi.useRealTimers();});+ it("rejects backfill ranges that start more than six months ago", () => {+ expect(() =>+ ActivationCostEstimatesSchema.parse({+ projectId: "project-id",+ evaluatorIds: ["evaluator-id"],+ filter: [],+ sampling: 1,+ timeRange: {+ from: new Date("2026-02-11T08:00:00.000Z"),+ to: new Date("2026-08-12T08:00:00.000Z"),+ },+ }),+ ).toThrow("The time range cannot start more than six months ago.");+ });+it("scopes reads to the project and estimates the sampled seven-day total", async () => {const result = await getActivationCostEstimates({orgId: "org-id",@@ -132,6 +148,43 @@ describe("getActivationCostEstimates", () => {]);});+ it("uses an explicit historical range for backfill estimates", async () => {+ const timeRange = {+ from: new Date("2026-05-01T00:00:00.000Z"),+ to: new Date("2026-08-01T23:59:59.999Z"),+ };++ await getActivationCostEstimates({+ orgId: "org-id",+ projectId: "project-id",+ evaluatorIds: ["evaluator-id"],+ filter: [],+ sampling: 0.5,+ shouldReadFromObservationsTable: false,+ timeRange,+ });++ expect(mocks.getObservationsCountFromEventsTable).toHaveBeenCalledWith({+ projectId: "project-id",+ filter: [+ {+ column: "startTime",+ type: "datetime",+ operator: ">=",+ value: timeRange.from,+ },+ {+ column: "startTime",+ type: "datetime",+ operator: "<=",+ value: timeRange.to,+ },+ ],+ limit: 1,+ offset: 0,+ });+ });+it("loads evaluators and counts matching observations once for the batch", async () => {mocks.findEvaluatorsByIds.mockResolvedValue([evaluator,diff --git a/web/src/features/evals/v2/server/evaluators/activationCostService.ts b/web/src/features/evals/v2/server/evaluators/activationCostService.tsindex 6fb1617398..61338b1d81 100644--- a/web/src/features/evals/v2/server/evaluators/activationCostService.ts+++ b/web/src/features/evals/v2/server/evaluators/activationCostService.ts@@ -31,6 +31,10 @@ export async function getActivationCostEstimates(params: {knownTestRunCostUsd?: number;shouldRunMissingTest?: boolean;shouldReadFromObservationsTable: boolean;+ timeRange?: {+ from: Date;+ to: Date;+ };}) {const validation = validateEvaluatorFiltersForTarget({targetObject: EvalTargetObject.EVENT,@@ -54,7 +58,7 @@ export async function getActivationCostEstimates(params: {(evaluator) => evaluator.type === EvalTemplateType.LLM_AS_JUDGE,);- const since = new Date(Date.now() - SEVEN_DAYS_MS);+ const since = params.timeRange?.from ?? new Date(Date.now() - SEVEN_DAYS_MS);const filter: FilterState = [...validation.validatedFilters,{@@ -63,6 +67,16 @@ export async function getActivationCostEstimates(params: {operator: ">=",value: since,},+ ...(params.timeRange+ ? [+ {+ column: "startTime" as const,+ type: "datetime" as const,+ operator: "<=" as const,+ value: params.timeRange.to,+ },+ ]+ : []),];const [matchingObservations, recentRunCosts] = await Promise.all([getObservationsCountFromEventsTable({diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorTypes.ts b/web/src/features/evals/v2/server/evaluators/evaluatorTypes.tsindex bc912c4049..c41c71492e 100644--- a/web/src/features/evals/v2/server/evaluators/evaluatorTypes.ts+++ b/web/src/features/evals/v2/server/evaluators/evaluatorTypes.ts@@ -14,6 +14,7 @@ import {type PersistedEvaluatorPromptMessages,} from "@langfuse/shared";import { z } from "zod";+import { endOfDay, startOfDay, subMonths } from "date-fns";const EvaluatorMetadataSchema = z.object({name: z.string().trim().min(1),@@ -144,6 +145,24 @@ export const ActivationCostEstimatesSchema = EvaluatorIdsSchema.extend({sampling: z.number().min(0).max(1),shouldRunMissingTest: z.boolean().optional().default(true),knownTestRunCostUsd: z.number().nonnegative().optional(),+ timeRange: z+ .object({+ from: z.date(),+ to: z.date(),+ })+ .refine(({ from, to }) => from <= to, {+ message: "The start of the time range must be before its end.",+ path: ["from"],+ })+ .refine(({ from }) => from >= startOfDay(subMonths(new Date(), 6)), {+ message: "The time range cannot start more than six months ago.",+ path: ["from"],+ })+ .refine(({ to }) => to <= endOfDay(new Date()), {+ message: "The time range cannot end in the future.",+ path: ["to"],+ })+ .optional(),}).refine(({ evaluatorIds }) => new Set(evaluatorIds).size === evaluatorIds.length,{ message: "Evaluator IDs must be unique", path: ["evaluatorIds"] },diff --git a/worker/src/__tests__/handleBatchActionJob-comment-filters.unit.test.ts b/worker/src/__tests__/handleBatchActionJob-comment-filters.unit.test.tsindex 97fdaf8c4b..a91d311cd2 100644--- a/worker/src/__tests__/handleBatchActionJob-comment-filters.unit.test.ts+++ b/worker/src/__tests__/handleBatchActionJob-comment-filters.unit.test.ts@@ -213,6 +213,8 @@ describe("event batch-action comment filter wiring", () => {...createPayload("observation-run-batched-evaluation"),evaluatorIds: ["evaluator-with-rule", "standalone-evaluator"],evalVersion: "v2",+ sampling: 0.25,+ rowLimit: 5_000,});expect(mocks.findEvaluators).toHaveBeenCalledWith(@@ -241,6 +243,12 @@ describe("event batch-action comment filter wiring", () => {],}),);+ expect(+ mocks.processBatchedObservationEval.mock.calls[0]?.[0].evaluators[0].sampling.toString(),+ ).toBe("0.25");+ expect(mocks.getEventsStreamForEval).toHaveBeenCalledWith(+ expect.objectContaining({ rowLimit: 5_000 }),+ );});it("applies mapping overrides to ruleless in-memory assignments", async () => {diff --git a/worker/src/features/batchAction/handleBatchActionJob.ts b/worker/src/features/batchAction/handleBatchActionJob.tsindex 317a45ccc8..21903bfcad 100644--- a/worker/src/features/batchAction/handleBatchActionJob.ts+++ b/worker/src/features/batchAction/handleBatchActionJob.ts@@ -477,6 +477,8 @@ export const handleBatchActionJob = async (batchActionId,evalVersion,evaluatorMappings,+ sampling = 1,+ rowLimit = env.LANGFUSE_MAX_HISTORIC_EVAL_CREATION_LIMIT,} = batchActionEvent;if (!batchActionId) {@@ -531,7 +533,7 @@ export const handleBatchActionJob = async (ruleId: null,projectId,filter: [] as [],- sampling: new Decimal(1),+ sampling: new Decimal(sampling),status: JobConfigState.ACTIVE,targetObject: EvalTargetObject.EVENT,assignments: [@@ -586,7 +588,7 @@ export const handleBatchActionJob = async (| typeof EvalTargetObject.EXPERIMENT,filter: [],}),- sampling: new Decimal(1),+ sampling: new Decimal(sampling),}));}} catch (error) {@@ -619,7 +621,10 @@ export const handleBatchActionJob = async (filter,searchQuery: query.searchQuery ?? undefined,searchType: query.searchType ?? ["id", "content"],- rowLimit: env.LANGFUSE_MAX_HISTORIC_EVAL_CREATION_LIMIT,+ rowLimit: Math.min(+ rowLimit,+ env.LANGFUSE_MAX_HISTORIC_EVAL_CREATION_LIMIT,+ ),});await processBatchedObservationEval({