Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(evals): backfill saved evaluators (#17071)

Tobias Wochingerc1b1b899b785
diff --git a/packages/shared/src/server/queues.ts b/packages/shared/src/server/queues.ts
index 6c87ba44f2..7d995ea025 100644
--- a/packages/shared/src/server/queues.ts
+++ b/packages/shared/src/server/queues.ts
@@ -269,6 +269,8 @@ export const BatchActionProcessingEventSchema = z.discriminatedUnion(
evaluatorIds: z.array(z.string()),
evalVersion: z.literal("v2").optional(),
evaluatorMappings: z.array(BatchEvalEvaluatorMappingSchema).optional(),
+ sampling: z.number().min(0).max(1).optional(),
+ rowLimit: z.number().int().positive().optional(),
}),
],
);
diff --git a/web/src/__tests__/server/unit/batch-actions-comment-filter.servertest.ts b/web/src/__tests__/server/unit/batch-actions-comment-filter.servertest.ts
index 447a2390f7..f8439c3a9f 100644
--- a/web/src/__tests__/server/unit/batch-actions-comment-filter.servertest.ts
+++ b/web/src/__tests__/server/unit/batch-actions-comment-filter.servertest.ts
@@ -317,6 +317,32 @@ describe("batched evaluation version selection", () => {
);
});
+ it("allows a capped batch when more observations match", async () => {
+ const context = prepare({ v4BetaEnabled: true });
+ mocks.getObservationsCountFromEventsTable.mockResolvedValue(100_000);
+
+ await context.runEvaluation.create({
+ projectId,
+ query,
+ evaluatorIds: [evaluatorId],
+ sourceTable: BatchEvalSourceTable.EVENTS,
+ evalVersion: "v2",
+ sampling: 0.25,
+ rowLimit: 5_000,
+ });
+
+ expect(mocks.queueAdd).toHaveBeenCalledWith(
+ expect.anything(),
+ expect.objectContaining({
+ payload: expect.objectContaining({
+ sampling: 0.25,
+ rowLimit: 5_000,
+ }),
+ }),
+ expect.anything(),
+ );
+ });
+
it("rejects evaluator v2 for users outside fast preview", async () => {
const context = prepare();
@@ -374,6 +400,8 @@ describe("batched evaluation version selection", () => {
sourceTable: BatchEvalSourceTable.EVENTS,
evalVersion: "v2",
evaluatorMappings,
+ sampling: 0.25,
+ rowLimit: 5_000,
});
expect(context.prisma.evaluator.findMany).toHaveBeenCalledWith(
@@ -390,6 +418,8 @@ describe("batched evaluation version selection", () => {
config: expect.objectContaining({
evalVersion: "v2",
evaluatorMappings,
+ sampling: 0.25,
+ rowLimit: 5_000,
}),
}),
}),
@@ -400,6 +430,8 @@ describe("batched evaluation version selection", () => {
payload: expect.objectContaining({
evalVersion: "v2",
evaluatorMappings,
+ sampling: 0.25,
+ rowLimit: 5_000,
}),
}),
expect.anything(),
diff --git a/web/src/features/batch-actions/server/runEvaluationRouter.ts b/web/src/features/batch-actions/server/runEvaluationRouter.ts
index 5e860abcc8..b75675cf90 100644
--- a/web/src/features/batch-actions/server/runEvaluationRouter.ts
+++ b/web/src/features/batch-actions/server/runEvaluationRouter.ts
@@ -42,6 +42,8 @@ export const runEvaluationRouter = createTRPCRouter({
evaluatorIds: rawEvaluatorIds,
sourceTable = BatchEvalSourceTable.EVENTS,
evaluatorMappings: rawEvaluatorMappings,
+ sampling,
+ rowLimit,
} = input;
if (env.LANGFUSE_MIGRATION_V4_ALLOW_PREVIEW_OPT_IN !== "true") {
@@ -145,7 +147,10 @@ export const runEvaluationRouter = createTRPCRouter({
? 0
: await getObservationsCountFromEventsTable(countQueryOpts);
- if (observationCount > env.LANGFUSE_MAX_HISTORIC_EVAL_CREATION_LIMIT) {
+ if (
+ rowLimit === undefined &&
+ observationCount > env.LANGFUSE_MAX_HISTORIC_EVAL_CREATION_LIMIT
+ ) {
throw new TRPCError({
code: "BAD_REQUEST",
message: `Too many observations selected. Maximum allowed is ${env.LANGFUSE_MAX_HISTORIC_EVAL_CREATION_LIMIT}, but ${observationCount} observations match your filters. Please refine your filters to reduce the count.`,
@@ -157,6 +162,8 @@ export const runEvaluationRouter = createTRPCRouter({
evaluatorIds,
...(input.evalVersion ? { evalVersion: input.evalVersion } : {}),
...(evaluatorMappings ? { evaluatorMappings } : {}),
+ ...(sampling !== undefined ? { sampling } : {}),
+ ...(rowLimit !== undefined ? { rowLimit } : {}),
};
logger.info(
@@ -206,6 +213,8 @@ export const runEvaluationRouter = createTRPCRouter({
? { evalVersion: batchConfig.evalVersion }
: {}),
...(evaluatorMappings ? { evaluatorMappings } : {}),
+ ...(sampling !== undefined ? { sampling } : {}),
+ ...(rowLimit !== undefined ? { rowLimit } : {}),
},
},
{
diff --git a/web/src/features/batch-actions/validation.ts b/web/src/features/batch-actions/validation.ts
index 96f8ce1b14..d4c8664a65 100644
--- a/web/src/features/batch-actions/validation.ts
+++ b/web/src/features/batch-actions/validation.ts
@@ -26,6 +26,8 @@ export const CreateObservationBatchEvaluationActionSchema = z
.array(BatchEvalEvaluatorMappingSchema)
.max(BATCH_EVAL_EVALUATOR_LIMIT)
.optional(),
+ sampling: z.number().min(0).max(1).optional(),
+ rowLimit: z.number().int().positive().max(25_000).optional(),
})
.superRefine((value, ctx) => {
if (!value.evaluatorMappings) return;
diff --git a/web/src/features/evals/v2/components/EvaluatorCostCalculationTooltipContent/EvaluatorCostCalculationTooltipContent.tsx b/web/src/features/evals/v2/components/EvaluatorCostCalculationTooltipContent/EvaluatorCostCalculationTooltipContent.tsx
new file mode 100644
index 0000000000..db6385c071
--- /dev/null
+++ b/web/src/features/evals/v2/components/EvaluatorCostCalculationTooltipContent/EvaluatorCostCalculationTooltipContent.tsx
@@ -0,0 +1,18 @@
+export function EvaluatorCostCalculationTooltipContent({
+ calculation,
+ explanation,
+}: {
+ calculation: string | null;
+ explanation: string;
+}) {
+ return (
+ <div className="space-y-2">
+ {calculation ? (
+ <p className="font-mono tabular-nums">{calculation}</p>
+ ) : null}
+ <p className={calculation ? "text-muted-foreground" : undefined}>
+ {explanation}
+ </p>
+ </div>
+ );
+}
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/EvaluatorBackfillSettings.stories.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/EvaluatorBackfillSettings.stories.tsx
new file mode 100644
index 0000000000..67bee2b7ea
--- /dev/null
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/EvaluatorBackfillSettings.stories.tsx
@@ -0,0 +1,85 @@
+import { useArgs } from "storybook/preview-api";
+import { fn } from "storybook/test";
+
+import preview from "../../../../../../../.storybook/preview";
+import { EvaluatorBackfillSettings } from "./EvaluatorBackfillSettings";
+
+const meta = preview.meta({ component: EvaluatorBackfillSettings });
+
+type EvaluatorBackfillSettingsProps = Parameters<
+ typeof EvaluatorBackfillSettings
+>[0];
+
+function StatefulEvaluatorBackfillSettings(
+ args: EvaluatorBackfillSettingsProps,
+) {
+ const [, updateArgs] = useArgs<EvaluatorBackfillSettingsProps>();
+
+ return (
+ <div className="max-w-xl rounded-md border p-3">
+ <EvaluatorBackfillSettings
+ {...args}
+ onEnabledChange={(enabled) => {
+ updateArgs({ enabled });
+ args.onEnabledChange(enabled);
+ }}
+ onWindowChange={(selectedWindow) => {
+ updateArgs({ selectedWindow });
+ args.onWindowChange(selectedWindow);
+ }}
+ onRangeChange={(range) => {
+ updateArgs({ range });
+ args.onRangeChange(range);
+ }}
+ onMaxItemsChange={(maxItems) => {
+ updateArgs({ maxItems });
+ args.onMaxItemsChange(maxItems);
+ }}
+ />
+ </div>
+ );
+}
+
+const sharedArgs: EvaluatorBackfillSettingsProps = {
+ enabled: true,
+ canEnable: true,
+ selectedWindow: "7-days",
+ range: {
+ from: new Date("2026-08-31T00:00:00"),
+ to: new Date("2026-09-07T23:59:59.999"),
+ },
+ maxItems: 5_000,
+ maxAllowedItems: 25_000,
+ matchingObservations: 4_400,
+ isEstimating: false,
+ onEnabledChange: fn(),
+ onWindowChange: fn(),
+ onRangeChange: fn(),
+ onMaxItemsChange: fn(),
+};
+
+export const LastSevenDays = meta.story({
+ args: sharedArgs,
+ render: StatefulEvaluatorBackfillSettings,
+});
+
+export const CustomRange = meta.story({
+ args: {
+ ...sharedArgs,
+ selectedWindow: "custom",
+ range: {
+ from: new Date("2026-08-01T00:00:00"),
+ to: new Date("2026-09-07T23:59:59.999"),
+ },
+ matchingObservations: 21_420,
+ },
+ render: StatefulEvaluatorBackfillSettings,
+});
+
+export const Disabled = meta.story({
+ args: {
+ ...sharedArgs,
+ enabled: false,
+ },
+ render: StatefulEvaluatorBackfillSettings,
+});
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/EvaluatorBackfillSettings.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/EvaluatorBackfillSettings.tsx
new file mode 100644
index 0000000000..58762d20c9
--- /dev/null
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/EvaluatorBackfillSettings.tsx
@@ -0,0 +1,196 @@
+import { format, subMonths } from "date-fns";
+import { Switch } from "@/src/components/design-system/Switch/Switch";
+import { DateRangeInput } from "@/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/components/DateRangeInput/DateRangeInput";
+import { Input } from "@/src/components/ui/input";
+import {
+ Select,
+ SelectContent,
+ SelectItem,
+ SelectTrigger,
+ SelectValue,
+} from "@/src/components/ui/select";
+import { numberFormatter } from "@/src/utils/numbers";
+import { cn } from "@/src/utils/tailwind";
+
+export type EvaluatorBackfillWindow =
+ | "24-hours"
+ | "7-days"
+ | "30-days"
+ | "90-days"
+ | "custom";
+
+export type EvaluatorBackfillRange = {
+ from: Date;
+ to: Date;
+};
+
+const windowOptions: Array<{
+ value: EvaluatorBackfillWindow;
+ label: string;
+ summaryLabel: string;
+}> = [
+ {
+ value: "24-hours",
+ label: "Last 24 hours",
+ summaryLabel: "the last 24 hours",
+ },
+ {
+ value: "7-days",
+ label: "Last 7 days",
+ summaryLabel: "the last 7 days",
+ },
+ {
+ value: "30-days",
+ label: "Last 30 days",
+ summaryLabel: "the last 30 days",
+ },
+ {
+ value: "90-days",
+ label: "Last 90 days",
+ summaryLabel: "the last 90 days",
+ },
+ {
+ value: "custom",
+ label: "Custom range",
+ summaryLabel: "the selected range",
+ },
+];
+
+export function EvaluatorBackfillSettings({
+ enabled,
+ canEnable,
+ selectedWindow,
+ range,
+ maxItems,
+ maxAllowedItems,
+ matchingObservations,
+ isEstimating,
+ onEnabledChange,
+ onWindowChange,
+ onRangeChange,
+ onMaxItemsChange,
+}: {
+ enabled: boolean;
+ canEnable: boolean;
+ selectedWindow: EvaluatorBackfillWindow;
+ range: EvaluatorBackfillRange;
+ maxItems: number;
+ maxAllowedItems: number;
+ matchingObservations: number;
+ isEstimating: boolean;
+ onEnabledChange: (enabled: boolean) => void;
+ onWindowChange: (window: EvaluatorBackfillWindow) => void;
+ onRangeChange: (range: EvaluatorBackfillRange) => void;
+ onMaxItemsChange: (maxItems: number) => void;
+}) {
+ const cappedCount = Math.min(matchingObservations, maxItems);
+ const rangeDays = Math.max(
+ 1,
+ Math.ceil(
+ (range.to.getTime() - range.from.getTime()) / (24 * 60 * 60 * 1_000),
+ ),
+ );
+ const selectedWindowOption = windowOptions.find(
+ (option) => option.value === selectedWindow,
+ );
+ const rangeDescription =
+ selectedWindow === "custom"
+ ? `the selected range (${rangeDays} days)`
+ : (selectedWindowOption?.summaryLabel ?? "the selected range");
+
+ return (
+ <section className="ml-6 max-w-full min-w-0">
+ <div className="flex min-w-0 items-center gap-2">
+ <Switch
+ id="evaluator-backfill-enabled"
+ size="sm"
+ checked={enabled}
+ disabled={!canEnable}
+ onCheckedChange={onEnabledChange}
+ />
+ <label
+ htmlFor="evaluator-backfill-enabled"
+ className={cn(
+ "min-w-0 text-sm leading-none",
+ canEnable ? "cursor-pointer" : "text-muted-foreground",
+ )}
+ >
+ Run evaluator on past observations matching these filters
+ </label>
+ </div>
+
+ {enabled ? (
+ <div className="mt-3 ml-9 max-w-full min-w-0 space-y-2">
+ <div className="flex min-w-0 flex-wrap items-center gap-2 text-sm">
+ <span>Score the newest</span>
+ <Input
+ id="evaluator-backfill-max-items"
+ aria-label="Max items"
+ type="number"
+ min={1}
+ max={maxAllowedItems}
+ className="w-20 text-right font-mono text-sm"
+ value={maxItems}
+ onChange={(event) => {
+ const value = Number(event.target.value);
+ if (!Number.isFinite(value)) return;
+ onMaxItemsChange(
+ Math.min(maxAllowedItems, Math.max(1, Math.floor(value))),
+ );
+ }}
+ />
+ <span>matches from the</span>
+ <Select
+ value={selectedWindow}
+ onValueChange={(value) =>
+ onWindowChange(value as EvaluatorBackfillWindow)
+ }
+ >
+ <SelectTrigger
+ aria-label="Backfill time window"
+ className="w-fit max-w-full whitespace-nowrap"
+ disableValueLineClamp
+ >
+ <SelectValue />
+ </SelectTrigger>
+ <SelectContent>
+ {windowOptions.map((option) => (
+ <SelectItem key={option.value} value={option.value}>
+ {option.label}
+ </SelectItem>
+ ))}
+ </SelectContent>
+ </Select>
+ </div>
+
+ {selectedWindow === "custom" ? (
+ <DateRangeInput
+ value={{
+ from: format(range.from, "yyyy-MM-dd"),
+ to: format(range.to, "yyyy-MM-dd"),
+ }}
+ min={format(subMonths(new Date(), 6), "yyyy-MM-dd")}
+ max={format(new Date(), "yyyy-MM-dd")}
+ fromAriaLabel="Backfill start date"
+ toAriaLabel="Backfill end date"
+ onValueChange={(value) =>
+ onRangeChange({
+ from: new Date(`${value.from}T00:00:00`),
+ to: new Date(`${value.to}T23:59:59.999`),
+ })
+ }
+ />
+ ) : null}
+
+ <p className="text-muted-foreground text-xs break-words">
+ {isEstimating
+ ? "Counting matching observations..."
+ : matchingObservations > maxItems
+ ? `${numberFormatter(matchingObservations, 0)} observations in ${rangeDescription} — capping at ${numberFormatter(cappedCount, 0)}, newest first.`
+ : `${numberFormatter(matchingObservations, 0)} observations in ${rangeDescription}, all within your limit.`}
+ </p>
+ </div>
+ ) : null}
+ </section>
+ );
+}
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/components/DateRangeInput/DateRangeInput.clienttest.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/components/DateRangeInput/DateRangeInput.clienttest.tsx
new file mode 100644
index 0000000000..236771097a
--- /dev/null
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/components/DateRangeInput/DateRangeInput.clienttest.tsx
@@ -0,0 +1,59 @@
+import { fireEvent, render, screen } from "@testing-library/react";
+
+import { DateRangeInput } from "./DateRangeInput";
+
+const value = {
+ from: "2026-08-31",
+ to: "2026-09-07",
+};
+
+describe("DateRangeInput", () => {
+ it("renders compact bounded date inputs", () => {
+ render(
+ <DateRangeInput
+ value={value}
+ min="2026-03-07"
+ max="2026-09-07"
+ onValueChange={vi.fn()}
+ />,
+ );
+
+ expect(screen.getByLabelText("Start date")).toHaveAttribute(
+ "min",
+ "2026-03-07",
+ );
+ expect(screen.getByLabelText("Start date")).toHaveAttribute(
+ "max",
+ value.to,
+ );
+ expect(screen.getByLabelText("End date")).toHaveAttribute(
+ "min",
+ value.from,
+ );
+ expect(screen.getByLabelText("End date")).toHaveAttribute(
+ "max",
+ "2026-09-07",
+ );
+ });
+
+ it("emits the complete range when either date changes", () => {
+ const onValueChange = vi.fn();
+ render(<DateRangeInput value={value} onValueChange={onValueChange} />);
+
+ fireEvent.change(screen.getByLabelText("Start date"), {
+ target: { value: "2026-08-30" },
+ });
+ expect(onValueChange).toHaveBeenLastCalledWith({
+ from: "2026-08-30",
+ to: value.to,
+ });
+
+ fireEvent.change(screen.getByLabelText("End date"), {
+ target: { value: "2026-09-06" },
+ });
+ expect(onValueChange).toHaveBeenLastCalledWith({
+ from: value.from,
+ to: "2026-09-06",
+ });
+ });
+});
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/components/DateRangeInput/DateRangeInput.stories.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/components/DateRangeInput/DateRangeInput.stories.tsx
new file mode 100644
index 0000000000..6053fda10e
--- /dev/null
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/components/DateRangeInput/DateRangeInput.stories.tsx
@@ -0,0 +1,59 @@
+import { useArgs } from "storybook/preview-api";
+import { fn } from "storybook/test";
+
+import preview from "../../../../../../../../../.storybook/preview";
+import { DateRangeInput } from "./DateRangeInput";
+
+const meta = preview.meta({ component: DateRangeInput });
+
+type DateRangeInputProps = Parameters<typeof DateRangeInput>[0];
+
+function StatefulDateRangeInput(args: DateRangeInputProps) {
+ const [, updateArgs] = useArgs<DateRangeInputProps>();
+
+ return (
+ <DateRangeInput
+ {...args}
+ onValueChange={(value) => {
+ updateArgs({ value });
+ args.onValueChange(value);
+ }}
+ />
+ );
+}
+
+export const Default = meta.story({
+ args: {
+ value: {
+ from: "2026-08-31",
+ to: "2026-09-07",
+ },
+ onValueChange: fn(),
+ },
+ render: StatefulDateRangeInput,
+});
+
+export const Bounded = meta.story({
+ args: {
+ value: {
+ from: "2026-08-31",
+ to: "2026-09-07",
+ },
+ min: "2026-03-07",
+ max: "2026-09-07",
+ onValueChange: fn(),
+ },
+ render: StatefulDateRangeInput,
+});
+
+export const Disabled = meta.story({
+ args: {
+ value: {
+ from: "2026-08-31",
+ to: "2026-09-07",
+ },
+ disabled: true,
+ onValueChange: fn(),
+ },
+ render: StatefulDateRangeInput,
+});
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/components/DateRangeInput/DateRangeInput.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/components/DateRangeInput/DateRangeInput.tsx
new file mode 100644
index 0000000000..bf3c2039d0
--- /dev/null
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/components/DateRangeInput/DateRangeInput.tsx
@@ -0,0 +1,58 @@
+import { Input } from "@/src/components/ui/input";
+
+export type DateInputRange = {
+ from: string;
+ to: string;
+};
+
+export function DateRangeInput({
+ value,
+ onValueChange,
+ min,
+ max,
+ fromAriaLabel = "Start date",
+ toAriaLabel = "End date",
+ disabled,
+}: {
+ value: DateInputRange;
+ onValueChange: (value: DateInputRange) => void;
+ min?: string;
+ max?: string;
+ fromAriaLabel?: string;
+ toAriaLabel?: string;
+ disabled?: boolean;
+}) {
+ return (
+ <div className="grid w-fit max-w-full grid-cols-[minmax(0,7.5rem)_auto_minmax(0,7.5rem)] items-center gap-2">
+ <Input
+ aria-label={fromAriaLabel}
+ type="date"
+ className="block min-w-0 [&::-webkit-calendar-picker-indicator]:translate-y-px"
+ value={value.from}
+ min={min}
+ max={value.to}
+ disabled={disabled}
+ onChange={(event) => {
+ if (!event.target.value) return;
+ onValueChange({ ...value, from: event.target.value });
+ }}
+ />
+ <span className="text-muted-foreground" aria-hidden="true">
+ →
+ </span>
+ <Input
+ aria-label={toAriaLabel}
+ type="date"
+ className="block min-w-0 [&::-webkit-calendar-picker-indicator]:translate-y-px"
+ value={value.to}
+ min={value.from}
+ max={max}
+ disabled={disabled}
+ onChange={(event) => {
+ if (!event.target.value) return;
+ onValueChange({ ...value, to: event.target.value });
+ }}
+ />
+ </div>
+ );
+}
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedCostSummary.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedCostSummary.tsx
index 544f4fa9fc..80e9debe2d 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedCostSummary.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedCostSummary.tsx
@@ -6,6 +6,7 @@ import {
SAMPLING_SLIDER_MIN,
SAMPLING_SLIDER_STEP,
} from "@/src/features/evals/v2/constants/ruleSampling";
+import { EvaluatorCostCalculationTooltipContent } from "@/src/features/evals/v2/components/EvaluatorCostCalculationTooltipContent/EvaluatorCostCalculationTooltipContent";
import type { ActivationEstimate } from "@/src/features/evals/v2/fns/requestRuleActivation";
import { formatEvaluatorCostCalculation } from "@/src/features/evals/v2/fns/formatEvaluatorCostCalculation";
import { compactNumberFormatter, usdFormatter } from "@/src/utils/numbers";
@@ -18,6 +19,7 @@ export function EvaluatorSavedCostSummary({
isEstimating,
onSamplingChange,
evaluatorType,
+ backfill,
}: {
estimates: ActivationEstimate[];
unavailableEstimateCount: number;
@@ -26,6 +28,15 @@ export function EvaluatorSavedCostSummary({
isEstimating: boolean;
onSamplingChange: ((sampling: number) => void) | null;
evaluatorType: EvalTemplateType;
+ backfill:
+ | { enabled: false }
+ | {
+ enabled: true;
+ matchingObservations: number;
+ maxItems: number;
+ testRunCostUsd: number | null;
+ isEstimating: boolean;
+ };
}) {
const estimate = estimates[0];
const sampledObservations = Math.round(matchingObservations * sampling);
@@ -35,6 +46,16 @@ export function EvaluatorSavedCostSummary({
: estimate
? estimate.matchingObservations * sampling * estimate.testRunCostUsd
: null;
+ const backfillObservationCount = backfill.enabled
+ ? Math.min(backfill.matchingObservations, backfill.maxItems)
+ : 0;
+ const backfillTestRunCostUsd = backfill.enabled
+ ? backfill.testRunCostUsd
+ : null;
+ const backfillEstimatedCostUsd =
+ backfillTestRunCostUsd === null
+ ? null
+ : backfillObservationCount * sampling * backfillTestRunCostUsd;
return (
<div className="space-y-5">
@@ -60,27 +81,29 @@ export function EvaluatorSavedCostSummary({
/>
</section>
- <section>
- <h3 className="text-sm font-bold">Matches</h3>
- {isEstimating ? (
- <div className="mt-2 space-y-2">
- <Skeleton className="h-5 w-24" />
- <Skeleton className="h-3 w-16" />
- </div>
- ) : (
- <>
- <p className="mt-1 font-mono text-base font-bold tabular-nums">
- {compactNumberFormatter(matchingObservations, 1)} / week
- </p>
- <p className="text-muted-foreground text-xs tabular-nums">
- {compactNumberFormatter(sampledObservations, 1)} sampled
- </p>
- </>
- )}
- </section>
+ {evaluatorType === EvalTemplateTypeEnum.CODE ? (
+ <section>
+ <h3 className="text-sm font-bold">Matches</h3>
+ {isEstimating ? (
+ <div className="mt-2 space-y-2">
+ <Skeleton className="h-5 w-24" />
+ <Skeleton className="h-3 w-16" />
+ </div>
+ ) : (
+ <>
+ <p className="mt-1 font-mono text-base font-bold tabular-nums">
+ {compactNumberFormatter(matchingObservations, 1)} / week
+ </p>
+ <p className="text-muted-foreground text-xs tabular-nums">
+ {compactNumberFormatter(sampledObservations, 1)} sampled
+ </p>
+ </>
+ )}
+ </section>
+ ) : null}
{evaluatorType !== EvalTemplateTypeEnum.CODE ? (
- <section className="border-t border-dashed pt-4">
+ <section className="border-t pt-4">
{isEstimating ? (
<div className="space-y-2">
<Skeleton className="h-6 w-24" />
@@ -88,6 +111,9 @@ export function EvaluatorSavedCostSummary({
</div>
) : (
<>
+ <p className="text-muted-foreground mb-1 font-mono text-[10px] tracking-wider uppercase">
+ Recurring
+ </p>
<div className="flex items-center gap-1.5">
<span className="font-mono text-lg font-bold tabular-nums">
{estimatedCostUsd === null
@@ -95,13 +121,15 @@ export function EvaluatorSavedCostSummary({
: `≈ ${usdFormatter(estimatedCostUsd, 2, 2)}`}
</span>
<InfoTooltip label="How estimated LLM costs are calculated">
- {formatEvaluatorCostCalculation({
- matchingObservations,
- sampling,
- testRunCostUsd: estimate?.testRunCostUsd ?? null,
- estimatedCostUsd,
- evaluatorType,
- })}
+ <EvaluatorCostCalculationTooltipContent
+ {...formatEvaluatorCostCalculation({
+ matchingObservations,
+ sampling,
+ testRunCostUsd: estimate?.testRunCostUsd ?? null,
+ estimatedCostUsd,
+ evaluatorType,
+ })}
+ />
</InfoTooltip>
</div>
<p className="text-muted-foreground text-xs">
@@ -117,6 +145,45 @@ export function EvaluatorSavedCostSummary({
)}
</section>
) : null}
+
+ {evaluatorType !== EvalTemplateTypeEnum.CODE && backfill.enabled ? (
+ <section className="border-t pt-4">
+ {backfill.isEstimating ? (
+ <div className="space-y-2">
+ <Skeleton className="h-6 w-24" />
+ <Skeleton className="h-3 w-20" />
+ </div>
+ ) : (
+ <>
+ <p className="text-muted-foreground mb-1 font-mono text-[10px] tracking-wider uppercase">
+ One-time backfill
+ </p>
+ <div className="flex items-center gap-1.5">
+ <span className="font-mono text-lg font-bold tabular-nums">
+ {backfillEstimatedCostUsd !== null
+ ? `≈ ${usdFormatter(backfillEstimatedCostUsd, 2, 2)}`
+ : "Unavailable"}
+ </span>
+ <InfoTooltip label="How estimated backfill costs are calculated">
+ <EvaluatorCostCalculationTooltipContent
+ {...formatEvaluatorCostCalculation({
+ matchingObservations: backfillObservationCount,
+ sampling,
+ testRunCostUsd: backfillTestRunCostUsd,
+ estimatedCostUsd: backfillEstimatedCostUsd,
+ evaluatorType,
+ period: "selection",
+ })}
+ />
+ </InfoTooltip>
+ </div>
+ <p className="text-muted-foreground text-xs">
+ estimated LLM costs, once
+ </p>
+ </>
+ )}
+ </section>
+ ) : null}
</div>
);
}
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedDialog.clienttest.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedDialog.clienttest.tsx
index ef35c74867..caf33c3935 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedDialog.clienttest.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedDialog.clienttest.tsx
@@ -5,6 +5,7 @@ import { EvaluatorSavedDialog } from "./EvaluatorSavedDialog";
const renderDialog = () => {
const onDismiss = vi.fn();
const onSecondaryAction = vi.fn();
+ const onModeChange = vi.fn();
render(
<TooltipProvider>
@@ -15,11 +16,12 @@ const renderDialog = () => {
"test-filters": null,
"different-scope": null,
}}
+ backfillContent={null}
costSummary={null}
canSubmit
isSubmitting={false}
primaryActionLabel="Execute"
- onModeChange={vi.fn()}
+ onModeChange={onModeChange}
onDismiss={onDismiss}
onSecondaryAction={onSecondaryAction}
onPrimaryAction={vi.fn()}
@@ -27,10 +29,22 @@ const renderDialog = () => {
</TooltipProvider>,
);
- return { onDismiss, onSecondaryAction };
+ return { onDismiss, onSecondaryAction, onModeChange };
};
describe("EvaluatorSavedDialog", () => {
+ it("selects an inactive mode when its card is clicked", () => {
+ const { onModeChange } = renderDialog();
+
+ fireEvent.click(
+ screen.getByText(
+ "Attach to a rule you already have, or create a new one.",
+ ),
+ );
+
+ expect(onModeChange).toHaveBeenCalledWith("different-scope");
+ });
+
it("treats the close button as a passive dismissal", () => {
const { onDismiss, onSecondaryAction } = renderDialog();
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedDialog.stories.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedDialog.stories.tsx
index 2b2762a881..d829f8ac4c 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedDialog.stories.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedDialog.stories.tsx
@@ -6,6 +6,7 @@ import { fn } from "storybook/test";
import { EvaluatorSavedCostSummary } from "./EvaluatorSavedCostSummary";
import { EvaluatorSavedDialog } from "./EvaluatorSavedDialog";
import { EvaluatorSavedRuleFilterPreview } from "./EvaluatorSavedRuleFilterPreview";
+import { EvaluatorBackfillSettings } from "../EvaluatorBackfillSettings/EvaluatorBackfillSettings";
const meta = preview.meta({ component: EvaluatorSavedDialog });
@@ -120,6 +121,25 @@ const unsupportedFilterModeContent = {
const sharedArgs = {
open: true,
modeContentByMode,
+ backfillContent: (
+ <EvaluatorBackfillSettings
+ enabled={false}
+ canEnable
+ selectedWindow="7-days"
+ range={{
+ from: new Date("2026-08-28T00:00:00"),
+ to: new Date("2026-09-04T23:59:59.999"),
+ }}
+ maxItems={5_000}
+ maxAllowedItems={25_000}
+ matchingObservations={4_400}
+ isEstimating={false}
+ onEnabledChange={fn()}
+ onWindowChange={fn()}
+ onRangeChange={fn()}
+ onMaxItemsChange={fn()}
+ />
+ ),
canSubmit: true,
isSubmitting: false,
onModeChange: fn(),
@@ -149,6 +169,7 @@ export const FromTestFilters = meta.story({
sampling={0.4}
isEstimating={false}
evaluatorType="LLM_AS_JUDGE"
+ backfill={{ enabled: false }}
onSamplingChange={fn()}
/>
),
@@ -156,6 +177,110 @@ export const FromTestFilters = meta.story({
},
});
+export const CodeEvaluator = meta.story({
+ args: {
+ ...sharedArgs,
+ mode: "test-filters",
+ costSummary: (
+ <EvaluatorSavedCostSummary
+ estimates={[]}
+ unavailableEstimateCount={0}
+ matchingObservations={80}
+ sampling={1}
+ isEstimating={false}
+ evaluatorType="CODE"
+ backfill={{ enabled: false }}
+ onSamplingChange={fn()}
+ />
+ ),
+ primaryActionLabel: "Execute",
+ },
+});
+
+const backfillCostSummary = (
+ <EvaluatorSavedCostSummary
+ estimates={[
+ {
+ evaluatorId: "evaluator-1",
+ evaluatorName: "Conciseness",
+ matchingObservations: 4_400,
+ sampling: 1,
+ testRunCostUsd: 0.0017,
+ estimatedCostUsd: 7.48,
+ },
+ ]}
+ unavailableEstimateCount={0}
+ matchingObservations={4_400}
+ sampling={1}
+ isEstimating={false}
+ evaluatorType="LLM_AS_JUDGE"
+ backfill={{
+ enabled: true,
+ matchingObservations: 4_400,
+ maxItems: 5_000,
+ testRunCostUsd: 0.0017,
+ isEstimating: false,
+ }}
+ onSamplingChange={fn()}
+ />
+);
+
+export const WithBackfill = meta.story({
+ args: {
+ ...sharedArgs,
+ mode: "test-filters",
+ backfillContent: (
+ <EvaluatorBackfillSettings
+ enabled
+ canEnable
+ selectedWindow="7-days"
+ range={{
+ from: new Date("2026-08-28T00:00:00"),
+ to: new Date("2026-09-04T23:59:59.999"),
+ }}
+ maxItems={5_000}
+ maxAllowedItems={25_000}
+ matchingObservations={4_400}
+ isEstimating={false}
+ onEnabledChange={fn()}
+ onWindowChange={fn()}
+ onRangeChange={fn()}
+ onMaxItemsChange={fn()}
+ />
+ ),
+ costSummary: backfillCostSummary,
+ primaryActionLabel: "Execute",
+ },
+});
+
+export const WithCustomBackfill = meta.story({
+ args: {
+ ...sharedArgs,
+ mode: "test-filters",
+ backfillContent: (
+ <EvaluatorBackfillSettings
+ enabled
+ canEnable
+ selectedWindow="custom"
+ range={{
+ from: new Date("2026-08-01T00:00:00"),
+ to: new Date("2026-09-04T23:59:59.999"),
+ }}
+ maxItems={5_000}
+ maxAllowedItems={25_000}
+ matchingObservations={21_420}
+ isEstimating={false}
+ onEnabledChange={fn()}
+ onWindowChange={fn()}
+ onRangeChange={fn()}
+ onMaxItemsChange={fn()}
+ />
+ ),
+ costSummary: backfillCostSummary,
+ primaryActionLabel: "Execute",
+ },
+});
+
export const UnsupportedFilters = meta.story({
args: {
...sharedArgs,
@@ -178,6 +303,7 @@ export const UnsupportedFilters = meta.story({
sampling={0.4}
isEstimating={false}
evaluatorType="LLM_AS_JUDGE"
+ backfill={{ enabled: false }}
onSamplingChange={fn()}
/>
),
@@ -206,6 +332,7 @@ export const ExistingRule = meta.story({
sampling={0.25}
isEstimating={false}
evaluatorType="LLM_AS_JUDGE"
+ backfill={{ enabled: false }}
onSamplingChange={null}
/>
),
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedDialog.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedDialog.tsx
index a5fdd50f16..91994f408c 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedDialog.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedDialog.tsx
@@ -41,6 +41,7 @@ export function EvaluatorSavedDialog({
open,
mode,
modeContentByMode,
+ backfillContent,
costSummary,
canSubmit,
isSubmitting,
@@ -49,11 +50,13 @@ export function EvaluatorSavedDialog({
onDismiss,
onSecondaryAction,
onPrimaryAction,
+ onOpenAutoFocus,
onCloseAnimationEnd,
}: {
open: boolean;
mode: EvaluatorSavedMode;
modeContentByMode: Record<EvaluatorSavedMode, ReactNode>;
+ backfillContent: ReactNode;
costSummary: ReactNode;
canSubmit: boolean;
isSubmitting: boolean;
@@ -62,6 +65,7 @@ export function EvaluatorSavedDialog({
onDismiss: () => void;
onSecondaryAction: () => void;
onPrimaryAction: () => void;
+ onOpenAutoFocus?: () => void;
onCloseAnimationEnd?: () => void;
}) {
return (
@@ -74,16 +78,17 @@ export function EvaluatorSavedDialog({
<DialogContent
className="sm:max-w-4xl"
closeOnInteractionOutside
+ onOpenAutoFocus={onOpenAutoFocus}
onCloseAutoFocus={onCloseAnimationEnd}
>
<DialogHeader className="[&>div]:items-start [&>div>button]:-mt-1">
<DialogTitle>Evaluator saved</DialogTitle>
<DialogDescription>
- Would you like to run this evaluator on incoming observations?
+ Choose which observations this evaluator should score.
</DialogDescription>
</DialogHeader>
<DialogBody className="gap-0 p-0">
- <div className="grid h-[22rem] grid-cols-[minmax(0,1fr)_15rem] overflow-hidden">
+ <div className="grid h-[30rem] grid-cols-[minmax(0,1fr)_15rem] overflow-hidden">
<div className="min-w-0 overflow-y-auto px-6 py-5 [scrollbar-gutter:stable]">
<h3 className="mb-2 text-sm font-bold">
Set up rule to run on incoming observations
@@ -103,11 +108,23 @@ export function EvaluatorSavedDialog({
<Collapsible key={option.value} open={selected} asChild>
<div
className={cn(
- "rounded-md border p-3 transition-colors duration-200",
+ "min-w-0 rounded-md border p-3 transition-colors duration-200",
selected
? "border-foreground bg-background"
- : "bg-muted/30 hover:bg-muted/50",
+ : "bg-muted/30 hover:bg-muted/50 cursor-pointer",
)}
+ onClick={(event) => {
+ if (selected) return;
+ const target = event.target as HTMLElement;
+ if (
+ target.closest(
+ "button, input, label, a, [role='radio']",
+ )
+ ) {
+ return;
+ }
+ onModeChange(option.value);
+ }}
>
<div className="flex items-start gap-2">
<div className="mt-0.5">
@@ -132,11 +149,12 @@ export function EvaluatorSavedDialog({
<CollapsibleContent
id={contentId}
aria-labelledby={id}
- className={styles.collapsibleContent}
+ className={cn(styles.collapsibleContent, "space-y-3")}
>
<div className="mt-3 ml-6 min-w-0 pr-1">
{modeContentByMode[option.value]}
</div>
+ {backfillContent}
</CollapsibleContent>
</div>
</Collapsible>
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsx
index 59cafeb9b7..359a2c54a0 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsx
@@ -6,7 +6,7 @@ import {
type ObservationVariableMapping,
} from "@langfuse/shared";
import { ChevronDown } from "lucide-react";
-import { useCallback, useEffect, useMemo, useRef, useState } from "react";
+import { useCallback, useMemo, useRef, useState } from "react";
import { Button } from "@/src/components/ui/button";
import { PopoverTrigger } from "@/src/components/ui/popover";
import { selectTriggerClassName } from "@/src/components/ui/select";
@@ -26,10 +26,22 @@ import { cn } from "@/src/utils/tailwind";
import { classifySampleFiltersForRule } from "@/src/features/evals/v2/fns/rules/classifySampleFiltersForRule";
import { getFilterAnalyticsProperties } from "@/src/features/evals/v2/fns/getFilterAnalyticsProperties";
import { EvaluatorSavedRuleFilterPreview } from "@/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialog/EvaluatorSavedRuleFilterPreview";
+import { EvaluatorBackfillSettings } from "@/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/EvaluatorBackfillSettings";
+import { useEvaluatorSavedBackfill } from "@/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/hooks/useEvaluatorSavedBackfill";
type Rule = RouterOutputs["evalsV2"]["rules"]["list"]["rules"][number];
type DialogPhase = "saved" | "closing-saved" | "create-rule" | "closed";
+type EvaluatorSavedDialogEvaluator = {
+ id: string;
+ name: string;
+ type: EvalTemplateType;
+ defaultVariableMapping: ObservationVariableMapping[];
+ sampleFilter: FilterState;
+ hasCompletedTestCall?: boolean;
+ testRunCostUsd?: number | null;
+};
+
export function EvaluatorSavedDialogContainer({
projectId,
evaluator,
@@ -37,15 +49,7 @@ export function EvaluatorSavedDialogContainer({
onFinish,
}: {
projectId: string;
- evaluator: {
- id: string;
- name: string;
- type: EvalTemplateType;
- defaultVariableMapping: ObservationVariableMapping[];
- sampleFilter: FilterState;
- hasCompletedTestCall?: boolean;
- testRunCostUsd?: number | null;
- };
+ evaluator: EvaluatorSavedDialogEvaluator;
onDismiss: () => Promise<void>;
onFinish: () => Promise<void>;
}) {
@@ -63,13 +67,21 @@ export function EvaluatorSavedDialogContainer({
>(undefined);
const [isEstimating, setIsEstimating] = useState(false);
const [testFilterSampling, setTestFilterSampling] = useState(1);
+ const [isCompleting, setIsCompleting] = useState(false);
const estimateRequestId = useRef(0);
- // Strict Mode replays mount effects; this mutation must run once per dialog.
const initialEstimateRequested = useRef(false);
const createRuleHandoffPending = useRef(false);
const hasRequestedMissingCostTest = useRef(
evaluator.hasCompletedTestCall ?? false,
);
+ const claimMissingCostTest = useCallback(() => {
+ if (hasRequestedMissingCostTest.current) return false;
+ hasRequestedMissingCostTest.current = true;
+ return true;
+ }, []);
+ const resetMissingCostTest = useCallback(() => {
+ hasRequestedMissingCostTest.current = false;
+ }, []);
const missingCostTestRequest = useRef<Promise<void> | null>(null);
const activeRules = api.evalsV2.rules.list.useQuery(
{
@@ -91,6 +103,17 @@ export function EvaluatorSavedDialogContainer({
},
{ enabled: dialogPhase === "saved" },
);
+ const historicEvaluationLimit = api.evals.globalJobConfigs.useQuery({
+ projectId,
+ });
+ const backfill = useEvaluatorSavedBackfill({
+ projectId,
+ evaluatorId: evaluator.id,
+ knownTestRunCostUsd: evaluator.testRunCostUsd ?? undefined,
+ historicEvaluationLimit: historicEvaluationLimit.data,
+ claimMissingCostTest,
+ resetMissingCostTest,
+ });
const availableRules = useMemo(
() =>
[
@@ -137,17 +160,35 @@ export function EvaluatorSavedDialogContainer({
};
const attachToRule = async (rule: Rule) => {
- await attach.mutateAsync({
+ const currentAssignments =
+ await utils.client.evalsV2.rules.listRulesForEvaluator.query({
+ projectId,
+ evaluatorId: evaluator.id,
+ });
+ const isAlreadyAttached = currentAssignments.some(
+ (assignment) => assignment.evaluationRule.id === rule.id,
+ );
+ if (!isAlreadyAttached) {
+ await attach.mutateAsync({
+ projectId,
+ ruleId: rule.id,
+ evaluatorId: evaluator.id,
+ variableMapping: null,
+ enableRule: !rule.enabled,
+ });
+ capture("evaluation_rules:attach_evaluator", {
+ evaluatorCount: 1,
+ source: "evaluator_create",
+ });
+ }
+ const currentRule = await utils.client.evalsV2.rules.get.query({
projectId,
ruleId: rule.id,
- evaluatorId: evaluator.id,
- variableMapping: null,
- enableRule: !rule.enabled,
- });
- capture("evaluation_rules:attach_evaluator", {
- evaluatorCount: 1,
- source: "evaluator_create",
});
+ await backfill.schedule(
+ { filter: currentRule.filter, sampling: currentRule.sampling },
+ backfill.executionRange,
+ );
await invalidateRuleQueries();
await finish();
};
@@ -174,6 +215,10 @@ export function EvaluatorSavedDialogContainer({
source: "evaluator_create_test_filters",
});
}
+ await backfill.schedule(
+ { filter: supportedRuleFilters, sampling },
+ backfill.executionRange,
+ );
await invalidateRuleQueries();
await finish();
};
@@ -192,10 +237,9 @@ export function EvaluatorSavedDialogContainer({
}
if (estimateRequestId.current !== requestId) return;
- const shouldRunMissingTest = !hasRequestedMissingCostTest.current;
+ const shouldRunMissingTest = claimMissingCostTest();
let finishMissingCostTestRequest: (() => void) | undefined;
if (shouldRunMissingTest) {
- hasRequestedMissingCostTest.current = true;
missingCostTestRequest.current = new Promise<void>((resolve) => {
finishMissingCostTestRequest = resolve;
});
@@ -231,42 +275,32 @@ export function EvaluatorSavedDialogContainer({
if (shouldRunMissingTest) missingCostTestRequest.current = null;
}
if (estimateRequestId.current !== requestId) return;
- if (result?.matchingObservations === 0) {
- hasRequestedMissingCostTest.current = false;
+ if (shouldRunMissingTest && result?.matchingObservations === 0) {
+ resetMissingCostTest();
}
} finally {
if (estimateRequestId.current === requestId) setIsEstimating(false);
}
},
[
+ claimMissingCostTest,
evaluator.id,
evaluator.name,
evaluator.testRunCostUsd,
requestActivation,
+ resetMissingCostTest,
setActivationOpen,
],
);
- useEffect(() => {
- if (
- dialogPhase !== "saved" ||
- mode !== "test-filters" ||
- initialEstimateRequested.current
- ) {
- return;
- }
+ const requestInitialEstimate = () => {
+ if (initialEstimateRequested.current) return;
initialEstimateRequested.current = true;
requestEstimate({
filter: supportedRuleFilters,
sampling: testFilterSampling,
}).catch(() => undefined);
- }, [
- dialogPhase,
- mode,
- requestEstimate,
- supportedRuleFilters,
- testFilterSampling,
- ]);
+ };
const openCreateRule = () => {
createRuleHandoffPending.current = true;
@@ -287,68 +321,115 @@ export function EvaluatorSavedDialogContainer({
requestEstimate({ filter: rule.filter, sampling: rule.sampling }).catch(
() => undefined,
);
+ if (backfill.enabled) {
+ backfill
+ .requestEstimate({ filter: rule.filter, sampling: rule.sampling })
+ .catch(() => undefined);
+ }
},
- [requestEstimate, setActivationSampling],
+ [backfill, requestEstimate, setActivationSampling],
);
const selectNewRule = () => {
estimateRequestId.current += 1;
setActivationOpen(false);
- setSelectedRuleId(null);
setIsEstimating(false);
+ setSelectedRuleId(null);
+ backfill.clearScope();
};
- useEffect(() => {
- if (
- mode !== "different-scope" ||
- selectedRuleId !== undefined ||
- rulesPending
- ) {
- return;
- }
-
- const mostUsedRule = availableRules[0];
- if (mostUsedRule) {
- selectExistingRule(mostUsedRule);
- } else {
- setSelectedRuleId(null);
- }
- }, [availableRules, mode, rulesPending, selectExistingRule, selectedRuleId]);
-
const handleModeChange = (nextMode: EvaluatorSavedMode) => {
- estimateRequestId.current += 1;
- if (nextMode !== "test-filters") initialEstimateRequested.current = false;
+ const modeChangeRequestId = ++estimateRequestId.current;
+ setActivationOpen(false);
+ setIsEstimating(false);
setMode(nextMode);
- const mostUsedRule = availableRules[0];
- if (nextMode === "different-scope" && mostUsedRule) {
- selectExistingRule(mostUsedRule);
+
+ if (nextMode === "different-scope") {
+ const selectMostUsedRule = (rules: Rule[]) => {
+ if (estimateRequestId.current !== modeChangeRequestId) return;
+ const mostUsedRule = rules[0];
+ if (mostUsedRule) {
+ selectExistingRule(mostUsedRule);
+ return;
+ }
+ setSelectedRuleId(null);
+ backfill.clearScope();
+ };
+
+ if (rulesPending) {
+ setSelectedRuleId(undefined);
+ Promise.all([activeRules.refetch(), inactiveRules.refetch()])
+ .then(([activeResult, inactiveResult]) =>
+ selectMostUsedRule(
+ [
+ ...(activeResult.data?.rules ?? []),
+ ...(inactiveResult.data?.rules ?? []),
+ ].sort(
+ (left, right) =>
+ right.assignments.length - left.assignments.length,
+ ),
+ ),
+ )
+ .catch(() => selectMostUsedRule([]));
+ return;
+ }
+
+ selectMostUsedRule(availableRules);
return;
}
- setSelectedRuleId(
- nextMode === "different-scope" && !rulesPending ? null : undefined,
- );
- setIsEstimating(false);
- if (nextMode === "test-filters") {
- setActivationSampling(testFilterSampling);
+
+ setSelectedRuleId(undefined);
+ setActivationSampling(testFilterSampling);
+ requestEstimate({
+ filter: supportedRuleFilters,
+ sampling: testFilterSampling,
+ }).catch(() => undefined);
+ if (backfill.enabled) {
+ backfill
+ .requestEstimate({
+ filter: supportedRuleFilters,
+ sampling: testFilterSampling,
+ })
+ .catch(() => undefined);
}
};
+ const currentBackfillScope =
+ mode === "test-filters"
+ ? { filter: supportedRuleFilters, sampling: testFilterSampling }
+ : selectedRule
+ ? { filter: selectedRule.filter, sampling: selectedRule.sampling }
+ : null;
+
const handlePrimaryAction = () => {
if (mode === "test-filters") {
capture("evaluators:saved_dialog_submit", {
action: "test_filters",
+ hasBackfill: backfill.enabled,
+ backfillWindow: backfill.enabled ? backfill.window : undefined,
+ backfillMaxItems: backfill.enabled
+ ? backfill.effectiveMaxItems
+ : undefined,
});
- resolveFromTestFilters().catch(() => undefined);
+ setIsCompleting(true);
+ resolveFromTestFilters().catch(() => setIsCompleting(false));
return;
}
if (selectedRule) {
capture("evaluators:saved_dialog_submit", {
action: "existing_rule",
+ hasBackfill: backfill.enabled,
+ backfillWindow: backfill.enabled ? backfill.window : undefined,
+ backfillMaxItems: backfill.enabled
+ ? backfill.effectiveMaxItems
+ : undefined,
});
- attachToRule(selectedRule).catch(() => undefined);
+ setIsCompleting(true);
+ attachToRule(selectedRule).catch(() => setIsCompleting(false));
} else {
capture("evaluators:saved_dialog_submit", {
action: "new_rule",
+ hasBackfill: false,
});
openCreateRule();
}
@@ -423,6 +504,28 @@ export function EvaluatorSavedDialogContainer({
};
const hasConfiguredScope = mode === "test-filters" || Boolean(selectedRule);
+ const backfillContent = (
+ <EvaluatorBackfillSettings
+ enabled={backfill.enabled}
+ canEnable={hasConfiguredScope}
+ selectedWindow={backfill.window}
+ range={backfill.range}
+ maxItems={backfill.effectiveMaxItems}
+ maxAllowedItems={backfill.allowedItems}
+ matchingObservations={backfill.matchingObservations}
+ isEstimating={backfill.isEstimating}
+ onEnabledChange={(enabled) =>
+ backfill.setEnabled(enabled, currentBackfillScope)
+ }
+ onWindowChange={(window) =>
+ backfill.updateWindow(window, currentBackfillScope)
+ }
+ onRangeChange={(range) =>
+ backfill.updateRange(range, currentBackfillScope)
+ }
+ onMaxItemsChange={backfill.setMaxItems}
+ />
+ );
const costSummary = hasConfiguredScope ? (
<EvaluatorSavedCostSummary
estimates={activation.estimate.estimates}
@@ -435,6 +538,17 @@ export function EvaluatorSavedDialogContainer({
}
isEstimating={isEstimating}
evaluatorType={evaluator.type}
+ backfill={
+ backfill.enabled
+ ? {
+ enabled: true,
+ matchingObservations: backfill.matchingObservations,
+ maxItems: backfill.effectiveMaxItems,
+ testRunCostUsd: backfill.testRunCostUsd,
+ isEstimating: backfill.isEstimating,
+ }
+ : { enabled: false }
+ }
onSamplingChange={
mode === "test-filters"
? (sampling) => {
@@ -458,13 +572,20 @@ export function EvaluatorSavedDialogContainer({
open={dialogPhase === "saved"}
mode={mode}
modeContentByMode={modeContentByMode}
+ backfillContent={backfillContent}
costSummary={costSummary}
canSubmit={
!isEstimating &&
+ !isCompleting &&
+ (!backfill.enabled || !backfill.isEstimating) &&
+ (!backfill.enabled || historicEvaluationLimit.isSuccess) &&
(mode === "test-filters" || selectedRuleId !== undefined)
}
isSubmitting={
- attach.isPending || createOrAttachFromEvaluatorFilters.isPending
+ attach.isPending ||
+ createOrAttachFromEvaluatorFilters.isPending ||
+ backfill.isScheduling ||
+ isCompleting
}
primaryActionLabel={
mode === "test-filters"
@@ -474,6 +595,7 @@ export function EvaluatorSavedDialogContainer({
: "Open rule editor"
}
onModeChange={handleModeChange}
+ onOpenAutoFocus={requestInitialEstimate}
onDismiss={() => {
createRuleHandoffPending.current = false;
setDialogPhase("closed");
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/hooks/useEvaluatorSavedBackfill.ts b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/hooks/useEvaluatorSavedBackfill.ts
new file mode 100644
index 0000000000..3d1a705540
--- /dev/null
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/hooks/useEvaluatorSavedBackfill.ts
@@ -0,0 +1,267 @@
+import { BatchEvalSourceTable, type FilterState } from "@langfuse/shared";
+import { endOfDay, startOfDay, subDays, subHours, subMonths } from "date-fns";
+import { useCallback, useRef, useState } from "react";
+import type {
+ EvaluatorBackfillRange,
+ EvaluatorBackfillWindow,
+} from "@/src/features/evals/v2/components/Evaluators/EvaluatorBackfillSettings/EvaluatorBackfillSettings";
+import {
+ DEFAULT_EVALUATOR_BACKFILL_ITEMS,
+ MAX_EVALUATOR_BACKFILL_ITEMS,
+} from "@/src/features/evals/v2/constants/evaluatorBackfill";
+import { api } from "@/src/utils/api";
+import { trpcErrorToast } from "@/src/utils/trpcErrorToast";
+
+type EvaluatorBackfillScope = {
+ filter: FilterState;
+ sampling: number;
+};
+
+function getBackfillRange(
+ window: Exclude<EvaluatorBackfillWindow, "custom">,
+ now = new Date(),
+): EvaluatorBackfillRange {
+ switch (window) {
+ case "24-hours":
+ return { from: subHours(now, 24), to: now };
+ case "7-days":
+ return { from: subDays(now, 7), to: now };
+ case "30-days":
+ return { from: subDays(now, 30), to: now };
+ case "90-days":
+ return { from: subDays(now, 90), to: now };
+ }
+}
+
+function normalizeBackfillRange(
+ nextRange: EvaluatorBackfillRange,
+ previousFrom: Date,
+ now = new Date(),
+): EvaluatorBackfillRange {
+ const earliestAllowedStart = startOfDay(subMonths(now, 6));
+ const latestAllowedEnd = endOfDay(now);
+ const clampDate = (date: Date) =>
+ new Date(
+ Math.min(
+ Math.max(date.getTime(), earliestAllowedStart.getTime()),
+ latestAllowedEnd.getTime(),
+ ),
+ );
+ let from = clampDate(nextRange.from);
+ let to = clampDate(nextRange.to);
+
+ if (from > to) {
+ if (nextRange.from.getTime() !== previousFrom.getTime()) {
+ to = clampDate(endOfDay(from));
+ } else {
+ from = clampDate(startOfDay(to));
+ }
+ }
+
+ return { from, to };
+}
+
+export function useEvaluatorSavedBackfill({
+ projectId,
+ evaluatorId,
+ knownTestRunCostUsd,
+ historicEvaluationLimit,
+ claimMissingCostTest,
+ resetMissingCostTest,
+}: {
+ projectId: string;
+ evaluatorId: string;
+ knownTestRunCostUsd?: number;
+ historicEvaluationLimit?: number;
+ claimMissingCostTest: () => boolean;
+ resetMissingCostTest: () => void;
+}) {
+ const utils = api.useUtils();
+ const runEvaluation = api.batchAction.runEvaluation.create.useMutation({
+ onError: trpcErrorToast,
+ });
+ const [enabled, setEnabledState] = useState(false);
+ const [window, setWindow] = useState<EvaluatorBackfillWindow>("7-days");
+ const [range, setRange] = useState<EvaluatorBackfillRange>(() =>
+ getBackfillRange("7-days"),
+ );
+ const [maxItems, setMaxItems] = useState(DEFAULT_EVALUATOR_BACKFILL_ITEMS);
+ const [matchingObservations, setMatchingObservations] = useState(0);
+ const [testRunCostUsd, setTestRunCostUsd] = useState<number | null>(null);
+ const [isEstimating, setIsEstimating] = useState(false);
+ const estimateRequestId = useRef(0);
+ const hasScheduled = useRef(false);
+
+ const requestEstimate = useCallback(
+ async (scope: EvaluatorBackfillScope, estimateRange = range) => {
+ const requestId = ++estimateRequestId.current;
+ setIsEstimating(true);
+ try {
+ const shouldRunMissingTest = claimMissingCostTest();
+ const result =
+ await utils.client.evalsV2.activationCostEstimates.mutate({
+ projectId,
+ evaluatorIds: [evaluatorId],
+ filter: scope.filter,
+ sampling: scope.sampling,
+ shouldRunMissingTest,
+ timeRange: estimateRange,
+ ...(knownTestRunCostUsd !== undefined
+ ? { knownTestRunCostUsd }
+ : {}),
+ });
+ if (estimateRequestId.current !== requestId) return;
+ const estimate = result.find(
+ ({ evaluatorId: resultEvaluatorId }) =>
+ resultEvaluatorId === evaluatorId,
+ );
+ if (shouldRunMissingTest && estimate?.matchingObservations === 0) {
+ resetMissingCostTest();
+ }
+ setMatchingObservations(estimate?.matchingObservations ?? 0);
+ setTestRunCostUsd(estimate?.testRunCostUsd ?? null);
+ } catch (error) {
+ if (estimateRequestId.current === requestId) {
+ setMatchingObservations(0);
+ setTestRunCostUsd(null);
+ trpcErrorToast(error);
+ }
+ } finally {
+ if (estimateRequestId.current === requestId) {
+ setIsEstimating(false);
+ }
+ }
+ },
+ [
+ claimMissingCostTest,
+ evaluatorId,
+ knownTestRunCostUsd,
+ projectId,
+ range,
+ resetMissingCostTest,
+ utils.client,
+ ],
+ );
+
+ const clearScope = useCallback(() => {
+ estimateRequestId.current += 1;
+ setIsEstimating(false);
+ setEnabledState(false);
+ setMatchingObservations(0);
+ setTestRunCostUsd(null);
+ }, []);
+
+ const setEnabled = useCallback(
+ (nextEnabled: boolean, scope: EvaluatorBackfillScope | null) => {
+ setEnabledState(nextEnabled);
+ if (!nextEnabled) {
+ estimateRequestId.current += 1;
+ setIsEstimating(false);
+ return;
+ }
+
+ const defaultWindow: EvaluatorBackfillWindow = "7-days";
+ const defaultRange = getBackfillRange(defaultWindow);
+ setWindow(defaultWindow);
+ setRange(defaultRange);
+ if (scope) {
+ requestEstimate(scope, defaultRange).catch(() => undefined);
+ }
+ },
+ [requestEstimate],
+ );
+
+ const updateRange = useCallback(
+ (
+ nextRange: EvaluatorBackfillRange,
+ scope: EvaluatorBackfillScope | null,
+ ) => {
+ const clampedRange = normalizeBackfillRange(nextRange, range.from);
+ setRange(clampedRange);
+ if (scope) {
+ requestEstimate(scope, clampedRange).catch(() => undefined);
+ }
+ },
+ [range.from, requestEstimate],
+ );
+
+ const updateWindow = useCallback(
+ (
+ nextWindow: EvaluatorBackfillWindow,
+ scope: EvaluatorBackfillScope | null,
+ ) => {
+ setWindow(nextWindow);
+ if (nextWindow !== "custom") {
+ updateRange(getBackfillRange(nextWindow), scope);
+ }
+ },
+ [updateRange],
+ );
+
+ const allowedItems = Math.min(
+ historicEvaluationLimit ?? MAX_EVALUATOR_BACKFILL_ITEMS,
+ MAX_EVALUATOR_BACKFILL_ITEMS,
+ );
+ const effectiveMaxItems = Math.min(maxItems, allowedItems);
+ const executionRange = window === "custom" ? range : getBackfillRange(window);
+
+ const schedule = async (
+ scope: EvaluatorBackfillScope,
+ executionRange: EvaluatorBackfillRange,
+ ) => {
+ if (!enabled || hasScheduled.current) return;
+ const normalizedExecutionRange = normalizeBackfillRange(
+ executionRange,
+ range.from,
+ );
+ await runEvaluation.mutateAsync({
+ projectId,
+ query: {
+ filter: [
+ ...scope.filter,
+ {
+ column: "startTime",
+ type: "datetime",
+ operator: ">=",
+ value: normalizedExecutionRange.from,
+ },
+ {
+ column: "startTime",
+ type: "datetime",
+ operator: "<=",
+ value: normalizedExecutionRange.to,
+ },
+ ],
+ orderBy: { column: "startTime", order: "DESC" },
+ useEventsTable: true,
+ },
+ evaluatorIds: [evaluatorId],
+ evaluatorMappings: [{ evaluatorId, variableMapping: null }],
+ evalVersion: "v2",
+ sourceTable: BatchEvalSourceTable.EVENTS,
+ sampling: scope.sampling,
+ rowLimit: effectiveMaxItems,
+ });
+ hasScheduled.current = true;
+ };
+
+ return {
+ enabled,
+ window,
+ range,
+ effectiveMaxItems,
+ allowedItems,
+ matchingObservations,
+ testRunCostUsd,
+ isEstimating,
+ isScheduling: runEvaluation.isPending,
+ executionRange,
+ requestEstimate,
+ clearScope,
+ setEnabled,
+ updateRange,
+ updateWindow,
+ setMaxItems,
+ schedule,
+ };
+}
diff --git a/web/src/features/evals/v2/components/Rules/RuleSetup/components/RuleEvaluatorCostEstimate.tsx b/web/src/features/evals/v2/components/Rules/RuleSetup/components/RuleEvaluatorCostEstimate.tsx
index 734581266a..f5e992529d 100644
--- a/web/src/features/evals/v2/components/Rules/RuleSetup/components/RuleEvaluatorCostEstimate.tsx
+++ b/web/src/features/evals/v2/components/Rules/RuleSetup/components/RuleEvaluatorCostEstimate.tsx
@@ -3,6 +3,7 @@ import {
TooltipContent,
TooltipTrigger,
} from "@/src/components/ui/tooltip";
+import { EvaluatorCostCalculationTooltipContent } from "@/src/features/evals/v2/components/EvaluatorCostCalculationTooltipContent/EvaluatorCostCalculationTooltipContent";
import { formatEvaluatorCostCalculation } from "@/src/features/evals/v2/fns/formatEvaluatorCostCalculation";
import type { RuleCostEstimate } from "@/src/features/evals/v2/hooks/useRuleCostEstimate";
import { usdFormatter } from "@/src/utils/numbers";
@@ -24,7 +25,9 @@ export function RuleEvaluatorCostEstimate({
</span>
</TooltipTrigger>
<TooltipContent className="max-w-72">
- {formatEvaluatorCostCalculation(estimate)}
+ <EvaluatorCostCalculationTooltipContent
+ {...formatEvaluatorCostCalculation(estimate)}
+ />
</TooltipContent>
</Tooltip>
);
diff --git a/web/src/features/evals/v2/constants/evaluatorBackfill.ts b/web/src/features/evals/v2/constants/evaluatorBackfill.ts
new file mode 100644
index 0000000000..8107d384de
--- /dev/null
+++ b/web/src/features/evals/v2/constants/evaluatorBackfill.ts
@@ -0,0 +1,2 @@
+export const MAX_EVALUATOR_BACKFILL_ITEMS = 25_000;
+export const DEFAULT_EVALUATOR_BACKFILL_ITEMS = 5_000;
diff --git a/web/src/features/evals/v2/fns/formatEvaluatorCostCalculation.clienttest.ts b/web/src/features/evals/v2/fns/formatEvaluatorCostCalculation.clienttest.ts
index 8d39a5b9eb..365799d888 100644
--- a/web/src/features/evals/v2/fns/formatEvaluatorCostCalculation.clienttest.ts
+++ b/web/src/features/evals/v2/fns/formatEvaluatorCostCalculation.clienttest.ts
@@ -13,9 +13,12 @@ describe("formatEvaluatorCostCalculation", () => {
estimatedCostUsd: 0.736,
evaluatorType: EvalTemplateTypeEnum.LLM_AS_JUDGE,
}),
- ).toBe(
- "1,840 matching observations × 40% sampling × $0.001 per evaluation = ≈ $0.74 / week. Expected cost on your linked API key (not Langfuse). Matching observations are based on the last 7 days. Per-evaluation cost uses the latest cost-bearing evaluator trace from that period, or a fallback test call.",
- );
+ ).toEqual({
+ calculation:
+ "1,840 matching observations × 40% sampling × $0.001 per evaluation ≈ $0.74 / week",
+ explanation:
+ "Expected cost on your linked API key (not Langfuse). Matching observations are based on the last 7 days. Per-evaluation cost uses the latest cost-bearing evaluator trace from that period, or a fallback test call.",
+ });
});
it("explains when neither a recent trace nor fallback test provides a cost", () => {
@@ -27,9 +30,11 @@ describe("formatEvaluatorCostCalculation", () => {
estimatedCostUsd: null,
evaluatorType: EvalTemplateTypeEnum.LLM_AS_JUDGE,
}),
- ).toBe(
- "No cost-bearing evaluator trace was available from the last 7 days, and the fallback test call did not return a usable model cost. Expected cost would be charged to your linked API key, not Langfuse.",
- );
+ ).toEqual({
+ calculation: null,
+ explanation:
+ "No cost-bearing evaluator trace was available from the last 7 days, and the fallback test call did not return a usable model cost. Expected cost would be charged to your linked API key, not Langfuse.",
+ });
});
it("shows zero cost when no observations match", () => {
@@ -41,9 +46,11 @@ describe("formatEvaluatorCostCalculation", () => {
estimatedCostUsd: 0,
evaluatorType: EvalTemplateTypeEnum.LLM_AS_JUDGE,
}),
- ).toBe(
- "0 matching observations × 100% sampling = ≈ $0.00 / week. Expected cost on your linked API key (not Langfuse). Matching observations are based on the last 7 days.",
- );
+ ).toEqual({
+ calculation: "0 matching observations × 100% sampling ≈ $0.00 / week",
+ explanation:
+ "Expected cost on your linked API key (not Langfuse). Matching observations are based on the last 7 days.",
+ });
});
it("explains why code evaluators have no model-provider cost", () => {
@@ -55,9 +62,11 @@ describe("formatEvaluatorCostCalculation", () => {
estimatedCostUsd: 0,
evaluatorType: EvalTemplateTypeEnum.CODE,
}),
- ).toBe(
- "Code evaluators do not call an LLM, so they do not incur model-provider / LLM costs.",
- );
+ ).toEqual({
+ calculation: null,
+ explanation:
+ "Code evaluators do not call an LLM, so they do not incur model-provider / LLM costs.",
+ });
});
it("does not claim a 7-day window when a selection has no priced test", () => {
@@ -70,9 +79,11 @@ describe("formatEvaluatorCostCalculation", () => {
evaluatorType: EvalTemplateTypeEnum.LLM_AS_JUDGE,
period: "selection",
}),
- ).toBe(
- "No cost-bearing evaluator trace was available, and the fallback test call did not return a usable model cost. Expected cost would be charged to your linked API key, not Langfuse.",
- );
+ ).toEqual({
+ calculation: null,
+ explanation:
+ "No cost-bearing evaluator trace was available, and the fallback test call did not return a usable model cost. Expected cost would be charged to your linked API key, not Langfuse.",
+ });
});
it("describes a one-shot selection instead of a weekly total", () => {
@@ -85,8 +96,11 @@ describe("formatEvaluatorCostCalculation", () => {
evaluatorType: EvalTemplateTypeEnum.LLM_AS_JUDGE,
period: "selection",
}),
- ).toBe(
- "12 observations × 100% sampling × $0.01 per evaluation = ≈ $0.12 for this run. Expected cost on your linked API key (not Langfuse). Per-evaluation cost uses the latest cost-bearing evaluator trace from that period, or a fallback test call.",
- );
+ ).toEqual({
+ calculation:
+ "12 observations × 100% sampling × $0.01 per evaluation ≈ $0.12 for this run",
+ explanation:
+ "Expected cost on your linked API key (not Langfuse). Per-evaluation cost uses the latest cost-bearing evaluator trace from that period, or a fallback test call.",
+ });
});
});
diff --git a/web/src/features/evals/v2/fns/formatEvaluatorCostCalculation.ts b/web/src/features/evals/v2/fns/formatEvaluatorCostCalculation.ts
index 6a4f8faa19..ba528ee951 100644
--- a/web/src/features/evals/v2/fns/formatEvaluatorCostCalculation.ts
+++ b/web/src/features/evals/v2/fns/formatEvaluatorCostCalculation.ts
@@ -17,7 +17,11 @@ export function formatEvaluatorCostCalculation({
period?: "week" | "selection";
}) {
if (evaluatorType === EvalTemplateTypeEnum.CODE) {
- return "Code evaluators do not call an LLM, so they do not incur model-provider / LLM costs.";
+ return {
+ calculation: null,
+ explanation:
+ "Code evaluators do not call an LLM, so they do not incur model-provider / LLM costs.",
+ };
}
const isSelection = period === "selection";
@@ -31,12 +35,21 @@ export function formatEvaluatorCostCalculation({
: "Expected cost on your linked API key (not Langfuse). Matching observations are based on the last 7 days.";
if (matchingObservations === 0) {
- return `${observations} × ${samplingRate} = ≈ ${usdFormatter(0, 2, 2)} ${periodLabel}. ${scope}`;
+ return {
+ calculation: `${observations} × ${samplingRate} ≈ ${usdFormatter(0, 2, 2)} ${periodLabel}`,
+ explanation: scope,
+ };
}
if (testRunCostUsd === null || estimatedCostUsd === null) {
const window = isSelection ? "" : " from the last 7 days";
- return `No cost-bearing evaluator trace was available${window}, and the fallback test call did not return a usable model cost. Expected cost would be charged to your linked API key, not Langfuse.`;
+ return {
+ calculation: null,
+ explanation: `No cost-bearing evaluator trace was available${window}, and the fallback test call did not return a usable model cost. Expected cost would be charged to your linked API key, not Langfuse.`,
+ };
}
- return `${observations} × ${samplingRate} × ${usdFormatter(testRunCostUsd, 2, 6)} per evaluation = ≈ ${usdFormatter(estimatedCostUsd, 2, 2)} ${periodLabel}. ${scope} Per-evaluation cost uses the latest cost-bearing evaluator trace from that period, or a fallback test call.`;
+ return {
+ calculation: `${observations} × ${samplingRate} × ${usdFormatter(testRunCostUsd, 2, 6)} per evaluation ≈ ${usdFormatter(estimatedCostUsd, 2, 2)} ${periodLabel}`,
+ explanation: `${scope} Per-evaluation cost uses the latest cost-bearing evaluator trace from that period, or a fallback test call.`,
+ };
}
diff --git a/web/src/features/evals/v2/server/evaluators/activationCostService.servertest.ts b/web/src/features/evals/v2/server/evaluators/activationCostService.servertest.ts
index 7bac13d266..94a729bad7 100644
--- a/web/src/features/evals/v2/server/evaluators/activationCostService.servertest.ts
+++ b/web/src/features/evals/v2/server/evaluators/activationCostService.servertest.ts
@@ -29,6 +29,7 @@ vi.mock("./testEvaluator", () => ({
}));
import { getActivationCostEstimates } from "./activationCostService";
+import { ActivationCostEstimatesSchema } from "./evaluatorTypes";
const evaluator = {
id: "evaluator-id",
@@ -78,6 +79,21 @@ describe("getActivationCostEstimates", () => {
vi.useRealTimers();
});
+ it("rejects backfill ranges that start more than six months ago", () => {
+ expect(() =>
+ ActivationCostEstimatesSchema.parse({
+ projectId: "project-id",
+ evaluatorIds: ["evaluator-id"],
+ filter: [],
+ sampling: 1,
+ timeRange: {
+ from: new Date("2026-02-11T08:00:00.000Z"),
+ to: new Date("2026-08-12T08:00:00.000Z"),
+ },
+ }),
+ ).toThrow("The time range cannot start more than six months ago.");
+ });
+
it("scopes reads to the project and estimates the sampled seven-day total", async () => {
const result = await getActivationCostEstimates({
orgId: "org-id",
@@ -132,6 +148,43 @@ describe("getActivationCostEstimates", () => {
]);
});
+ it("uses an explicit historical range for backfill estimates", async () => {
+ const timeRange = {
+ from: new Date("2026-05-01T00:00:00.000Z"),
+ to: new Date("2026-08-01T23:59:59.999Z"),
+ };
+
+ await getActivationCostEstimates({
+ orgId: "org-id",
+ projectId: "project-id",
+ evaluatorIds: ["evaluator-id"],
+ filter: [],
+ sampling: 0.5,
+ shouldReadFromObservationsTable: false,
+ timeRange,
+ });
+
+ expect(mocks.getObservationsCountFromEventsTable).toHaveBeenCalledWith({
+ projectId: "project-id",
+ filter: [
+ {
+ column: "startTime",
+ type: "datetime",
+ operator: ">=",
+ value: timeRange.from,
+ },
+ {
+ column: "startTime",
+ type: "datetime",
+ operator: "<=",
+ value: timeRange.to,
+ },
+ ],
+ limit: 1,
+ offset: 0,
+ });
+ });
+
it("loads evaluators and counts matching observations once for the batch", async () => {
mocks.findEvaluatorsByIds.mockResolvedValue([
evaluator,
diff --git a/web/src/features/evals/v2/server/evaluators/activationCostService.ts b/web/src/features/evals/v2/server/evaluators/activationCostService.ts
index 6fb1617398..61338b1d81 100644
--- a/web/src/features/evals/v2/server/evaluators/activationCostService.ts
+++ b/web/src/features/evals/v2/server/evaluators/activationCostService.ts
@@ -31,6 +31,10 @@ export async function getActivationCostEstimates(params: {
knownTestRunCostUsd?: number;
shouldRunMissingTest?: boolean;
shouldReadFromObservationsTable: boolean;
+ timeRange?: {
+ from: Date;
+ to: Date;
+ };
}) {
const validation = validateEvaluatorFiltersForTarget({
targetObject: EvalTargetObject.EVENT,
@@ -54,7 +58,7 @@ export async function getActivationCostEstimates(params: {
(evaluator) => evaluator.type === EvalTemplateType.LLM_AS_JUDGE,
);
- const since = new Date(Date.now() - SEVEN_DAYS_MS);
+ const since = params.timeRange?.from ?? new Date(Date.now() - SEVEN_DAYS_MS);
const filter: FilterState = [
...validation.validatedFilters,
{
@@ -63,6 +67,16 @@ export async function getActivationCostEstimates(params: {
operator: ">=",
value: since,
},
+ ...(params.timeRange
+ ? [
+ {
+ column: "startTime" as const,
+ type: "datetime" as const,
+ operator: "<=" as const,
+ value: params.timeRange.to,
+ },
+ ]
+ : []),
];
const [matchingObservations, recentRunCosts] = await Promise.all([
getObservationsCountFromEventsTable({
diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorTypes.ts b/web/src/features/evals/v2/server/evaluators/evaluatorTypes.ts
index bc912c4049..c41c71492e 100644
--- a/web/src/features/evals/v2/server/evaluators/evaluatorTypes.ts
+++ b/web/src/features/evals/v2/server/evaluators/evaluatorTypes.ts
@@ -14,6 +14,7 @@ import {
type PersistedEvaluatorPromptMessages,
} from "@langfuse/shared";
import { z } from "zod";
+import { endOfDay, startOfDay, subMonths } from "date-fns";
const EvaluatorMetadataSchema = z.object({
name: z.string().trim().min(1),
@@ -144,6 +145,24 @@ export const ActivationCostEstimatesSchema = EvaluatorIdsSchema.extend({
sampling: z.number().min(0).max(1),
shouldRunMissingTest: z.boolean().optional().default(true),
knownTestRunCostUsd: z.number().nonnegative().optional(),
+ timeRange: z
+ .object({
+ from: z.date(),
+ to: z.date(),
+ })
+ .refine(({ from, to }) => from <= to, {
+ message: "The start of the time range must be before its end.",
+ path: ["from"],
+ })
+ .refine(({ from }) => from >= startOfDay(subMonths(new Date(), 6)), {
+ message: "The time range cannot start more than six months ago.",
+ path: ["from"],
+ })
+ .refine(({ to }) => to <= endOfDay(new Date()), {
+ message: "The time range cannot end in the future.",
+ path: ["to"],
+ })
+ .optional(),
}).refine(
({ evaluatorIds }) => new Set(evaluatorIds).size === evaluatorIds.length,
{ message: "Evaluator IDs must be unique", path: ["evaluatorIds"] },
diff --git a/worker/src/__tests__/handleBatchActionJob-comment-filters.unit.test.ts b/worker/src/__tests__/handleBatchActionJob-comment-filters.unit.test.ts
index 97fdaf8c4b..a91d311cd2 100644
--- a/worker/src/__tests__/handleBatchActionJob-comment-filters.unit.test.ts
+++ b/worker/src/__tests__/handleBatchActionJob-comment-filters.unit.test.ts
@@ -213,6 +213,8 @@ describe("event batch-action comment filter wiring", () => {
...createPayload("observation-run-batched-evaluation"),
evaluatorIds: ["evaluator-with-rule", "standalone-evaluator"],
evalVersion: "v2",
+ sampling: 0.25,
+ rowLimit: 5_000,
});
expect(mocks.findEvaluators).toHaveBeenCalledWith(
@@ -241,6 +243,12 @@ describe("event batch-action comment filter wiring", () => {
],
}),
);
+ expect(
+ mocks.processBatchedObservationEval.mock.calls[0]?.[0].evaluators[0].sampling.toString(),
+ ).toBe("0.25");
+ expect(mocks.getEventsStreamForEval).toHaveBeenCalledWith(
+ expect.objectContaining({ rowLimit: 5_000 }),
+ );
});
it("applies mapping overrides to ruleless in-memory assignments", async () => {
diff --git a/worker/src/features/batchAction/handleBatchActionJob.ts b/worker/src/features/batchAction/handleBatchActionJob.ts
index 317a45ccc8..21903bfcad 100644
--- a/worker/src/features/batchAction/handleBatchActionJob.ts
+++ b/worker/src/features/batchAction/handleBatchActionJob.ts
@@ -477,6 +477,8 @@ export const handleBatchActionJob = async (
batchActionId,
evalVersion,
evaluatorMappings,
+ sampling = 1,
+ rowLimit = env.LANGFUSE_MAX_HISTORIC_EVAL_CREATION_LIMIT,
} = batchActionEvent;
if (!batchActionId) {
@@ -531,7 +533,7 @@ export const handleBatchActionJob = async (
ruleId: null,
projectId,
filter: [] as [],
- sampling: new Decimal(1),
+ sampling: new Decimal(sampling),
status: JobConfigState.ACTIVE,
targetObject: EvalTargetObject.EVENT,
assignments: [
@@ -586,7 +588,7 @@ export const handleBatchActionJob = async (
| typeof EvalTargetObject.EXPERIMENT,
filter: [],
}),
- sampling: new Decimal(1),
+ sampling: new Decimal(sampling),
}));
}
} catch (error) {
@@ -619,7 +621,10 @@ export const handleBatchActionJob = async (
filter,
searchQuery: query.searchQuery ?? undefined,
searchType: query.searchType ?? ["id", "content"],
- rowLimit: env.LANGFUSE_MAX_HISTORIC_EVAL_CREATION_LIMIT,
+ rowLimit: Math.min(
+ rowLimit,
+ env.LANGFUSE_MAX_HISTORIC_EVAL_CREATION_LIMIT,
+ ),
});
await processBatchedObservationEval({