Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(evals): allow overriding variable mapping in batch evaluations (#16975)

Tobias Wochingerc02addc81928
diff --git a/packages/shared/src/features/evals/types.ts b/packages/shared/src/features/evals/types.ts
index e43a4df01c..6edc40ec5b 100644
--- a/packages/shared/src/features/evals/types.ts
+++ b/packages/shared/src/features/evals/types.ts
@@ -317,3 +317,16 @@ export const observationVariableMappingList = z.array(
export type ObservationVariableMapping = z.infer<
typeof observationVariableMapping
>;
+
+/**
+ * Per-evaluator mapping override for a one-shot batch evaluation. `null`
+ * inherits the evaluator version's mapping. Absent from the payload means
+ * every selected evaluator inherits.
+ */
+export const BatchEvalEvaluatorMappingSchema = z.object({
+ evaluatorId: z.string().min(1),
+ variableMapping: observationVariableMappingList.nullable(),
+});
+export type BatchEvalEvaluatorMapping = z.infer<
+ typeof BatchEvalEvaluatorMappingSchema
+>;
diff --git a/packages/shared/src/server/queues.ts b/packages/shared/src/server/queues.ts
index 845f4e0b1e..6c87ba44f2 100644
--- a/packages/shared/src/server/queues.ts
+++ b/packages/shared/src/server/queues.ts
@@ -9,7 +9,11 @@ import { BatchTableNames } from "../interfaces/tableNames";
import { EventActionSchema } from "../domain";
import { PromptDomainSchema } from "../domain/prompts";
import { ObservationAddToDatasetConfigSchema } from "../features/batchAction/addToDatasetTypes";
-import { EvalTargetObjectSchema } from "../features/evals/types";
+import {
+ BatchEvalEvaluatorMappingSchema,
+ EvalTargetObjectSchema,
+ observationVariableMappingList,
+} from "../features/evals/types";
import { EvalExecutionMode } from "../features/evals/evalConfigBlocking";
import {
type MonitorQueueEvent,
@@ -157,6 +161,10 @@ export const ObservationEvalExecutionEventSchema = z.object({
// which the executor resolves on pickup and records on the execution.
evaluatorId: z.string().optional(),
evaluationRuleId: z.string().optional(),
+ // Ruleless manual batch runs have no assignment row to hold a mapping
+ // override. Optional for jobs queued before this field existed; those
+ // inherit the evaluator version mapping.
+ variableMapping: observationVariableMappingList.optional(),
});
export const PostHogIntegrationProcessingEventSchema = z.object({
projectId: z.string(),
@@ -260,6 +268,7 @@ export const BatchActionProcessingEventSchema = z.discriminatedUnion(
batchActionId: z.string(),
evaluatorIds: z.array(z.string()),
evalVersion: z.literal("v2").optional(),
+ evaluatorMappings: z.array(BatchEvalEvaluatorMappingSchema).optional(),
}),
],
);
diff --git a/web/src/__tests__/server/unit/batch-actions-comment-filter.servertest.ts b/web/src/__tests__/server/unit/batch-actions-comment-filter.servertest.ts
index 3f55c9be01..447a2390f7 100644
--- a/web/src/__tests__/server/unit/batch-actions-comment-filter.servertest.ts
+++ b/web/src/__tests__/server/unit/batch-actions-comment-filter.servertest.ts
@@ -94,6 +94,7 @@ const session = {
function prepare({
v4BetaEnabled = false,
foundEvaluatorIds = [evaluatorId],
+ missingPromptVariable = false,
} = {}) {
const batchActionCreate = vi
.fn()
@@ -107,7 +108,37 @@ function prepare({
findMany: vi.fn(async () => [{ id: evaluatorId }]),
},
evaluator: {
- findMany: vi.fn(async () => foundEvaluatorIds.map((id) => ({ id }))),
+ findMany: vi.fn(async (args?: { select?: { id?: boolean } }) => {
+ if (args?.select?.id) {
+ return foundEvaluatorIds.map((id) => ({ id }));
+ }
+ return foundEvaluatorIds.map((id) => ({
+ id,
+ name: "Quality",
+ type: "LLM_AS_JUDGE",
+ versions: [
+ {
+ prompt: missingPromptVariable
+ ? "Evaluate {{output}} {{input}}"
+ : "Evaluate {{output}}",
+ promptMessages: [
+ {
+ role: "user",
+ content: missingPromptVariable
+ ? "Evaluate {{output}} {{input}}"
+ : "Evaluate {{output}}",
+ },
+ ],
+ variableMapping: [
+ {
+ templateVariable: "output",
+ selectedColumnId: "output",
+ },
+ ],
+ },
+ ],
+ }));
+ }),
},
} as unknown as PrismaClient;
const ctx = {
@@ -324,4 +355,143 @@ describe("batched evaluation version selection", () => {
expect(context.batchActionCreate).not.toHaveBeenCalled();
expect(mocks.queueAdd).not.toHaveBeenCalled();
});
+
+ it("queues mapping overrides on the batch-eval payload", async () => {
+ const context = prepare({ v4BetaEnabled: true });
+ const evaluatorMappings = [
+ {
+ evaluatorId,
+ variableMapping: [
+ { templateVariable: "output", selectedColumnId: "input" },
+ ],
+ },
+ ];
+
+ await context.runEvaluation.create({
+ projectId,
+ query,
+ evaluatorIds: [evaluatorId],
+ sourceTable: BatchEvalSourceTable.EVENTS,
+ evalVersion: "v2",
+ evaluatorMappings,
+ });
+
+ expect(context.prisma.evaluator.findMany).toHaveBeenCalledWith(
+ expect.objectContaining({ select: { id: true } }),
+ );
+ expect(context.prisma.evaluator.findMany).toHaveBeenCalledWith(
+ expect.objectContaining({
+ include: expect.objectContaining({ versions: expect.anything() }),
+ }),
+ );
+ expect(context.batchActionCreate).toHaveBeenCalledWith(
+ expect.objectContaining({
+ data: expect.objectContaining({
+ config: expect.objectContaining({
+ evalVersion: "v2",
+ evaluatorMappings,
+ }),
+ }),
+ }),
+ );
+ expect(mocks.queueAdd).toHaveBeenCalledWith(
+ expect.anything(),
+ expect.objectContaining({
+ payload: expect.objectContaining({
+ evalVersion: "v2",
+ evaluatorMappings,
+ }),
+ }),
+ expect.anything(),
+ );
+ });
+
+ it("rejects mapping overrides without evaluator v2", async () => {
+ const context = prepare({ v4BetaEnabled: true });
+
+ await expect(
+ context.runEvaluation.create({
+ projectId,
+ query,
+ evaluatorIds: [evaluatorId],
+ sourceTable: BatchEvalSourceTable.EVENTS,
+ evaluatorMappings: [
+ {
+ evaluatorId,
+ variableMapping: [
+ { templateVariable: "output", selectedColumnId: "output" },
+ ],
+ },
+ ],
+ }),
+ ).rejects.toMatchObject({ code: "BAD_REQUEST" });
+
+ expect(context.batchActionCreate).not.toHaveBeenCalled();
+ expect(mocks.queueAdd).not.toHaveBeenCalled();
+ });
+
+ it("rejects a mapping override for an unselected evaluator", async () => {
+ const context = prepare({ v4BetaEnabled: true });
+
+ await expect(
+ context.runEvaluation.create({
+ projectId,
+ query,
+ evaluatorIds: [evaluatorId],
+ sourceTable: BatchEvalSourceTable.EVENTS,
+ evalVersion: "v2",
+ evaluatorMappings: [
+ {
+ evaluatorId: "other-evaluator",
+ variableMapping: null,
+ },
+ ],
+ }),
+ ).rejects.toMatchObject({ code: "BAD_REQUEST" });
+
+ expect(context.batchActionCreate).not.toHaveBeenCalled();
+ expect(mocks.queueAdd).not.toHaveBeenCalled();
+ });
+
+ it("rejects more than 100 evaluators", async () => {
+ const context = prepare({ v4BetaEnabled: true });
+
+ await expect(
+ context.runEvaluation.create({
+ projectId,
+ query,
+ evaluatorIds: Array.from(
+ { length: 101 },
+ (_, index) => `evaluator-${index}`,
+ ),
+ sourceTable: BatchEvalSourceTable.EVENTS,
+ evalVersion: "v2",
+ }),
+ ).rejects.toMatchObject({ code: "BAD_REQUEST" });
+
+ expect(context.batchActionCreate).not.toHaveBeenCalled();
+ });
+
+ it("names the evaluator when a mapping is incomplete", async () => {
+ const context = prepare({
+ v4BetaEnabled: true,
+ missingPromptVariable: true,
+ });
+
+ await expect(
+ context.runEvaluation.create({
+ projectId,
+ query,
+ evaluatorIds: [evaluatorId],
+ sourceTable: BatchEvalSourceTable.EVENTS,
+ evalVersion: "v2",
+ evaluatorMappings: [{ evaluatorId, variableMapping: null }],
+ }),
+ ).rejects.toMatchObject({
+ code: "BAD_REQUEST",
+ message: expect.stringContaining('Evaluator "Quality"'),
+ });
+
+ expect(context.batchActionCreate).not.toHaveBeenCalled();
+ });
});
diff --git a/web/src/features/batch-actions/components/RunEvaluationDialog/ConfirmationStep.tsx b/web/src/features/batch-actions/components/RunEvaluationDialog/ConfirmationStep.tsx
index f5fb951a1f..4ba16dbb8d 100644
--- a/web/src/features/batch-actions/components/RunEvaluationDialog/ConfirmationStep.tsx
+++ b/web/src/features/batch-actions/components/RunEvaluationDialog/ConfirmationStep.tsx
@@ -2,7 +2,8 @@ import { Card, CardContent } from "@/src/components/ui/card";
import { Badge } from "@/src/components/ui/badge";
import { Separator } from "@/src/components/ui/separator";
import { EstimatedCostRow } from "./EstimatedCostRow";
-import { BatchEvalSourceTable } from "@langfuse/shared";
+import type { BatchEvalSourceTable } from "@langfuse/shared";
+import { getBatchEvalCostObservationCount } from "./utils";
type ConfirmationStepProps = {
projectId: string;
@@ -10,53 +11,15 @@ type ConfirmationStepProps = {
evaluators: Array<{ id: string; name: string }>;
hideCount: boolean;
sourceTable: BatchEvalSourceTable;
- experimentCount?: number;
};
-/**
- * Determines whether to show a cost disclaimer instead of the actual cost estimate.
- * For experiments source, we can't accurately estimate cost because displayCount
- * is the experiment count, not the actual observation count.
- */
-function shouldShowCostDisclaimer(sourceTable: BatchEvalSourceTable): boolean {
- return sourceTable === BatchEvalSourceTable.EXPERIMENTS;
-}
-
-/**
- * Calculates the effective observation count for cost estimation.
- * For experiment-items source, multiplies by experiment count since each item
- * is evaluated once per experiment.
- */
-function getEffectiveObservationCount(
- displayCount: number,
- sourceTable: BatchEvalSourceTable,
- experimentCount?: number,
-): number {
- if (
- sourceTable === BatchEvalSourceTable.EXPERIMENT_ITEMS &&
- experimentCount
- ) {
- return displayCount * experimentCount;
- }
- return displayCount;
-}
-
export function ConfirmationStep(props: ConfirmationStepProps) {
- const {
- projectId,
- displayCount,
- evaluators,
- hideCount,
- sourceTable,
- experimentCount,
- } = props;
+ const { projectId, displayCount, evaluators, hideCount, sourceTable } = props;
- const showCostDisclaimer = shouldShowCostDisclaimer(sourceTable);
- const effectiveObservationCount = getEffectiveObservationCount(
+ const effectiveObservationCount = getBatchEvalCostObservationCount({
displayCount,
sourceTable,
- experimentCount,
- );
+ });
return (
<div className="space-y-4">
@@ -86,7 +49,7 @@ export function ConfirmationStep(props: ConfirmationStepProps) {
<Separator />
- {showCostDisclaimer ? (
+ {effectiveObservationCount == null ? (
<div className="flex gap-2">
<span className="text-muted-foreground shrink-0">
Est. LLM API Key Cost:
diff --git a/web/src/features/batch-actions/components/RunEvaluationDialog/EvaluatorMappingStep.tsx b/web/src/features/batch-actions/components/RunEvaluationDialog/EvaluatorMappingStep.tsx
new file mode 100644
index 0000000000..863c868f45
--- /dev/null
+++ b/web/src/features/batch-actions/components/RunEvaluationDialog/EvaluatorMappingStep.tsx
@@ -0,0 +1,226 @@
+import { useStore } from "zustand";
+import type { ReactNode } from "react";
+
+import { Card, CardContent } from "@/src/components/ui/card";
+import { InfoTooltip } from "@/src/components/ui/InfoTooltip/InfoTooltip";
+import { Skeleton } from "@/src/components/ui/skeleton";
+import {
+ EvaluatorAssignmentsEditor,
+ type RuleEvaluatorOption,
+} from "@/src/features/evals";
+import {
+ useRuleCostEstimate,
+ type RuleCostEstimate,
+} from "@/src/features/evals/v2/hooks/useRuleCostEstimate";
+import type { RuleSetupStore } from "@/src/features/evals/v2/types/rules";
+import { usdFormatter } from "@/src/utils/numbers";
+
+export function EvaluatorMappingStep({
+ projectId,
+ store,
+ evaluatorOptions,
+ isQueryLoading,
+ isQueryError,
+ queryErrorMessage,
+ search,
+ onSearchChange,
+ sampleObject,
+ costObservationCount,
+}: {
+ projectId: string;
+ store: RuleSetupStore;
+ evaluatorOptions: RuleEvaluatorOption[];
+ isQueryLoading: boolean;
+ isQueryError: boolean;
+ queryErrorMessage: string | undefined;
+ search: string;
+ onSearchChange: (value: string) => void;
+ sampleObject: Record<string, unknown> | null;
+ costObservationCount: number | null;
+}) {
+ if (costObservationCount == null) {
+ return (
+ <EvaluatorMappingEditor
+ store={store}
+ evaluatorOptions={evaluatorOptions}
+ isQueryLoading={isQueryLoading}
+ isQueryError={isQueryError}
+ queryErrorMessage={queryErrorMessage}
+ search={search}
+ onSearchChange={onSearchChange}
+ sampleObject={sampleObject}
+ costEstimates={[]}
+ estimatingEvaluatorIds={[]}
+ footerTrailing={
+ <p className="text-muted-foreground max-w-64 text-right text-xs">
+ Cost estimate unavailable for experiment-scoped evaluations
+ </p>
+ }
+ costError={false}
+ />
+ );
+ }
+
+ return (
+ <EvaluatorMappingStepWithCosts
+ projectId={projectId}
+ store={store}
+ evaluatorOptions={evaluatorOptions}
+ isQueryLoading={isQueryLoading}
+ isQueryError={isQueryError}
+ queryErrorMessage={queryErrorMessage}
+ search={search}
+ onSearchChange={onSearchChange}
+ sampleObject={sampleObject}
+ costObservationCount={costObservationCount}
+ />
+ );
+}
+
+function EvaluatorMappingStepWithCosts({
+ projectId,
+ store,
+ evaluatorOptions,
+ isQueryLoading,
+ isQueryError,
+ queryErrorMessage,
+ search,
+ onSearchChange,
+ sampleObject,
+ costObservationCount,
+}: {
+ projectId: string;
+ store: RuleSetupStore;
+ evaluatorOptions: RuleEvaluatorOption[];
+ isQueryLoading: boolean;
+ isQueryError: boolean;
+ queryErrorMessage: string | undefined;
+ search: string;
+ onSearchChange: (value: string) => void;
+ sampleObject: Record<string, unknown> | null;
+ costObservationCount: number;
+}) {
+ const assignments = useStore(store, (state) => state.assignments);
+ const costEstimate = useRuleCostEstimate({
+ projectId,
+ store,
+ matchingObservations: costObservationCount,
+ });
+ const availableEstimates = costEstimate.estimates.filter(
+ (estimate) => estimate.estimatedCostUsd !== null,
+ );
+ const totalCostUsd = availableEstimates.reduce(
+ (total, estimate) => total + (estimate.estimatedCostUsd ?? 0),
+ 0,
+ );
+ const footerTrailing =
+ assignments.length > 0 ? (
+ <div className="w-48 border-t pt-2 text-right text-sm">
+ {availableEstimates.length > 0 ? (
+ <div>
+ <div className="flex items-center justify-end gap-1.5">
+ <p className="font-mono leading-none font-bold whitespace-nowrap tabular-nums">
+ ≈ {usdFormatter(totalCostUsd, 2, 2)}
+ </p>
+ <InfoTooltip label="About total estimated LLM costs">
+ Sum of the available LLM cost estimates for this run.
+ </InfoTooltip>
+ </div>
+ <p className="text-muted-foreground text-xs whitespace-nowrap">
+ estimated LLM costs for this run
+ </p>
+ </div>
+ ) : costEstimate.status === "estimating" ? (
+ <Skeleton className="ml-auto h-5 w-28" />
+ ) : (
+ <div>
+ <p className="font-mono font-bold">Unavailable</p>
+ <p className="text-muted-foreground text-xs whitespace-nowrap">
+ estimated LLM costs for this run
+ </p>
+ </div>
+ )}
+ </div>
+ ) : null;
+
+ return (
+ <EvaluatorMappingEditor
+ store={store}
+ evaluatorOptions={evaluatorOptions}
+ isQueryLoading={isQueryLoading}
+ isQueryError={isQueryError}
+ queryErrorMessage={queryErrorMessage}
+ search={search}
+ onSearchChange={onSearchChange}
+ sampleObject={sampleObject}
+ costEstimates={costEstimate.estimates}
+ estimatingEvaluatorIds={costEstimate.estimatingEvaluatorIds}
+ footerTrailing={footerTrailing}
+ costError={costEstimate.status === "error" && assignments.length > 0}
+ />
+ );
+}
+
+function EvaluatorMappingEditor({
+ store,
+ evaluatorOptions,
+ isQueryLoading,
+ isQueryError,
+ queryErrorMessage,
+ search,
+ onSearchChange,
+ sampleObject,
+ costEstimates,
+ estimatingEvaluatorIds,
+ footerTrailing,
+ costError,
+}: {
+ store: RuleSetupStore;
+ evaluatorOptions: RuleEvaluatorOption[];
+ isQueryLoading: boolean;
+ isQueryError: boolean;
+ queryErrorMessage: string | undefined;
+ search: string;
+ onSearchChange: (value: string) => void;
+ sampleObject: Record<string, unknown> | null;
+ costEstimates: RuleCostEstimate[];
+ estimatingEvaluatorIds: string[];
+ footerTrailing: ReactNode;
+ costError: boolean;
+}) {
+ return (
+ <div className="flex h-full min-h-0 flex-col gap-3">
+ <div className="min-h-0 flex-1 overflow-y-auto">
+ {isQueryLoading ? (
+ <p className="text-muted-foreground text-sm">Loading evaluators...</p>
+ ) : isQueryError ? (
+ <Card>
+ <CardContent className="text-destructive p-4 text-sm">
+ Failed to load evaluators: {queryErrorMessage}
+ </CardContent>
+ </Card>
+ ) : (
+ <>
+ <EvaluatorAssignmentsEditor
+ evaluatorOptions={evaluatorOptions}
+ store={store}
+ search={search}
+ onSearchChange={onSearchChange}
+ sampleObject={sampleObject}
+ costEstimates={costEstimates}
+ estimatingEvaluatorIds={estimatingEvaluatorIds}
+ footerTrailing={footerTrailing}
+ emptyDescription="Attach an evaluator to score this selection."
+ sourceUnavailableMessage="No observation is available to validate JSON paths."
+ />
+ {costError ? (
+ <p className="text-muted-foreground mt-2 text-xs">
+ Cost estimates are temporarily unavailable.
+ </p>
+ ) : null}
+ </>
+ )}
+ </div>
+ </div>
+ );
+}
diff --git a/web/src/features/batch-actions/components/RunEvaluationDialog/EvaluatorSelectionStep.tsx b/web/src/features/batch-actions/components/RunEvaluationDialog/EvaluatorSelectionStep.tsx
index dd8b29dd03..b11bf31f94 100644
--- a/web/src/features/batch-actions/components/RunEvaluationDialog/EvaluatorSelectionStep.tsx
+++ b/web/src/features/batch-actions/components/RunEvaluationDialog/EvaluatorSelectionStep.tsx
@@ -1,5 +1,4 @@
import { useMemo } from "react";
-import Link from "next/link";
import {
observationVariableMappingList,
type ObservationVariableMapping,
@@ -12,7 +11,7 @@ import { Checkbox } from "@/src/components/design-system/Checkbox/Checkbox";
import { Input } from "@/src/components/ui/input";
import { EvaluatorPromptPreview } from "./EvaluatorPromptPreview";
import { renderPromptPreviewFromObservation } from "./utils";
-import { ExternalLink, Eye, Plus, X } from "lucide-react";
+import { Eye, X } from "lucide-react";
export type BatchEvaluator = {
id: string;
@@ -34,7 +33,6 @@ type EvaluatorSelectionStepProps = {
evaluatorSearchQuery: string;
onSearchQueryChange: (query: string) => void;
onToggleEvaluator: (evaluatorId: string) => void;
- createEvaluatorHref: string;
};
export function EvaluatorSelectionStep(props: EvaluatorSelectionStepProps) {
@@ -50,7 +48,6 @@ export function EvaluatorSelectionStep(props: EvaluatorSelectionStepProps) {
evaluatorSearchQuery,
onSearchQueryChange,
onToggleEvaluator,
- createEvaluatorHref,
} = props;
const filteredEvaluators = useMemo(() => {
@@ -228,19 +225,6 @@ export function EvaluatorSelectionStep(props: EvaluatorSelectionStepProps) {
</div>
)}
</div>
-
- <Button variant="outline" size="default" className="h-9 w-full" asChild>
- <Link
- href={createEvaluatorHref}
- target="_blank"
- rel="noreferrer"
- aria-label="Create new Evaluator (opens in a new tab)"
- >
- <Plus className="mr-1 h-4 w-4" />
- Create new Evaluator
- <ExternalLink className="ml-1 h-3.5 w-3.5" aria-hidden="true" />
- </Link>
- </Button>
</div>
);
}
diff --git a/web/src/features/batch-actions/components/RunEvaluationDialog/index.tsx b/web/src/features/batch-actions/components/RunEvaluationDialog/index.tsx
index 6f0ef133c3..683ce4fd2f 100644
--- a/web/src/features/batch-actions/components/RunEvaluationDialog/index.tsx
+++ b/web/src/features/batch-actions/components/RunEvaluationDialog/index.tsx
@@ -1,8 +1,11 @@
import { useMemo, useState } from "react";
+import Link from "next/link";
+import { useStore } from "zustand";
import {
type BatchActionQuery,
type BatchEvalSourceTable,
BatchEvalSourceTable as SourceTable,
+ extractVariables,
observationVariableMappingList,
} from "@langfuse/shared";
import { api, sendAsPostOption } from "@/src/utils/api";
@@ -16,15 +19,35 @@ import {
DialogTitle,
} from "@/src/components/ui/dialog";
import { Button } from "@/src/components/ui/button";
+import { Skeleton } from "@/src/components/ui/skeleton";
+import {
+ Tooltip,
+ TooltipContent,
+ TooltipTrigger,
+} from "@/src/components/ui/tooltip";
import { showErrorToast, showSuccessToast } from "@/src/features/notifications";
-import { ChevronLeft } from "lucide-react";
+import { ChevronLeft, ExternalLink, Plus } from "lucide-react";
import {
EvaluatorSelectionStep,
type BatchEvaluator,
} from "./EvaluatorSelectionStep";
+import { EvaluatorMappingStep } from "./EvaluatorMappingStep";
import { ConfirmationStep } from "./ConfirmationStep";
-import { buildQueryWithSelectedIds, getCreateEvaluatorHref } from "./utils";
-import { useForceV3Experience } from "@/src/features/v4-migration/useForceV3Experience";
+import {
+ buildQueryWithSelectedIds,
+ getBatchEvalCostObservationCount,
+ getCreateEvaluatorHref,
+ hasCompleteBatchEvalMappings,
+} from "./utils";
+import { BATCH_EVAL_EVALUATOR_LIMIT } from "@/src/features/batch-actions/validation";
+import {
+ buildSelectedSampleObject,
+ createRuleSetupStore,
+} from "@/src/features/evals";
+import { coverEvaluatorPromptVariables } from "@/src/features/evals/v2/fns/variableMapping/coverEvaluatorPromptVariables";
+import { prepareModernRuleVariableMapping } from "@/src/features/evals/v2/fns/variableMapping/prepareModernRuleVariableMapping";
+import { useDebounce } from "@/src/hooks/useDebounce";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics";
type RunEvaluationDialogProps = {
projectId: string;
@@ -44,9 +67,6 @@ type RunEvaluationDialogProps = {
type DialogStep = "select-evaluator" | "confirm";
-/** Matches the evaluator overview page size; the step filters client-side. */
-const BATCH_EVALUATOR_LIMIT = 100;
-
export function RunEvaluationDialog(props: RunEvaluationDialogProps) {
const {
projectId,
@@ -57,20 +77,53 @@ export function RunEvaluationDialog(props: RunEvaluationDialogProps) {
sourceTable = SourceTable.EVENTS,
} = props;
+ const capture = usePostHogClientCapture();
const [step, setStep] = useState<DialogStep>("select-evaluator");
const [selectedEvaluators, setSelectedEvaluators] = useState<
BatchEvaluator[]
>([]);
const [evaluatorSearchQuery, setEvaluatorSearchQuery] = useState("");
- const forceV3Experience = useForceV3Experience(projectId);
+ const [mappingSearch, setMappingSearch] = useState("");
+ const [mappingSearchQuery, setMappingSearchQuery] = useState("");
+ const debouncedMappingSearch = useDebounce(setMappingSearchQuery, 300, false);
+ const [ruleSetupStore] = useState(() =>
+ createRuleSetupStore({
+ name: "Batch evaluation",
+ filter: [],
+ sampling: 1,
+ assignments: [],
+ }),
+ );
+ const mappingAssignments = useStore(
+ ruleSetupStore,
+ (state) => state.assignments,
+ );
- // Unsearched: `EvaluatorSelectionStep` filters the list client-side, the
- // same way the overview does, so typing does not refetch.
- const evaluatorsQuery = api.evalsV2.options.useQuery({
- projectId,
- limit: BATCH_EVALUATOR_LIMIT,
- excludeLegacyEvaluators: true,
- });
+ const forceV3Query = api.v4Transition.forceV3Experience.useQuery(
+ { projectId },
+ {
+ enabled: Boolean(projectId),
+ staleTime: Infinity,
+ },
+ );
+ const forceV3Experience = forceV3Query.data === true;
+ const isExperiencePending = forceV3Query.isPending;
+ const showMappingEditor = !isExperiencePending && !forceV3Experience;
+
+ const evaluatorsQuery = api.evalsV2.options.useQuery(
+ {
+ projectId,
+ limit: BATCH_EVAL_EVALUATOR_LIMIT,
+ excludeLegacyEvaluators: true,
+ search:
+ showMappingEditor && mappingSearchQuery.trim()
+ ? mappingSearchQuery.trim()
+ : undefined,
+ },
+ {
+ placeholderData: (previousData) => previousData,
+ },
+ );
const runEvaluationMutation =
api.batchAction.runEvaluation.create.useMutation({
@@ -80,7 +133,6 @@ export function RunEvaluationDialog(props: RunEvaluationDialogProps) {
});
const displayCount = selectAll ? totalCount : selectedObservationIds.length;
- // For experiments source, displayCount is experiment count, not item count
const isExperimentsSource = sourceTable === SourceTable.EXPERIMENTS;
const scopeLabel =
sourceTable === SourceTable.EVENTS ? "observation" : "experiment item";
@@ -115,8 +167,6 @@ export function RunEvaluationDialog(props: RunEvaluationDialogProps) {
const eligibleEvaluators = useMemo(
() =>
- // A blocked evaluator is skipped by the scheduler, so offering it would
- // report a successful batch that produced no scores.
(evaluatorsQuery.data ?? [])
.filter((evaluator) => evaluator.blockedAt === null)
.map(
@@ -133,11 +183,62 @@ export function RunEvaluationDialog(props: RunEvaluationDialogProps) {
[evaluatorsQuery.data],
);
+ const evaluatorOptions = useMemo(
+ () =>
+ (evaluatorsQuery.data ?? [])
+ .filter((evaluator) => evaluator.blockedAt === null)
+ .map((evaluator) => {
+ const prepared = prepareModernRuleVariableMapping(
+ evaluator.latestVersion?.variableMapping,
+ evaluator.type,
+ );
+ const requiredVariables =
+ evaluator.type === "CODE"
+ ? []
+ : extractVariables(evaluator.latestVersion?.prompt ?? "");
+
+ return {
+ id: evaluator.id,
+ name: evaluator.name,
+ type: evaluator.type,
+ updatedAt: evaluator.updatedAt,
+ createdByUser: evaluator.createdByUser,
+ defaultVariableMapping: coverEvaluatorPromptVariables(
+ prepared.defaultVariableMapping,
+ requiredVariables,
+ ),
+ initialVariableMapping: prepared.initialVariableMapping,
+ requiredVariables,
+ };
+ }),
+ [evaluatorsQuery.data],
+ );
+
const selectedEvaluatorIds = useMemo(
() => selectedEvaluators.map((evaluator) => evaluator.id),
[selectedEvaluators],
);
+ const sampleObject = buildSelectedSampleObject({
+ observation: props.exampleObservation?.id ? props.exampleObservation : null,
+ eventDetails: previewQuery.data?.[0],
+ });
+
+ const selectedCount = showMappingEditor
+ ? mappingAssignments.length
+ : selectedEvaluators.length;
+ const mappingsComplete = hasCompleteBatchEvalMappings(mappingAssignments);
+ const costObservationCount = getBatchEvalCostObservationCount({
+ displayCount,
+ sourceTable,
+ });
+ const mappingRunDisabledReason =
+ selectedCount === 0
+ ? "Attach at least one evaluator."
+ : mappingsComplete
+ ? null
+ : "Map every evaluator variable to a source column before running.";
+
const toggleEvaluatorSelection = (evaluatorId: string) => {
setSelectedEvaluators((previous) => {
if (previous.some((evaluator) => evaluator.id === evaluatorId)) {
@@ -151,7 +252,10 @@ export function RunEvaluationDialog(props: RunEvaluationDialogProps) {
};
const onSubmit = async () => {
- if (selectedEvaluators.length === 0) {
+ if (selectedCount === 0) {
+ return;
+ }
+ if (showMappingEditor && !mappingsComplete) {
return;
}
@@ -160,26 +264,47 @@ export function RunEvaluationDialog(props: RunEvaluationDialogProps) {
selectAll,
selectedObservationIds,
});
+ const evaluatorIds = showMappingEditor
+ ? mappingAssignments.map((assignment) => assignment.evaluatorId)
+ : selectedEvaluators.map((evaluator) => evaluator.id);
+ const evaluatorMappings = showMappingEditor
+ ? mappingAssignments.map((assignment) => ({
+ evaluatorId: assignment.evaluatorId,
+ variableMapping: assignment.variableMapping,
+ }))
+ : undefined;
+ const mappingOverrideCount = (evaluatorMappings ?? []).filter(
+ (mapping) => mapping.variableMapping !== null,
+ ).length;
try {
await runEvaluationMutation.mutateAsync({
projectId,
query: finalQuery,
- evaluatorIds: selectedEvaluators.map((evaluator) => evaluator.id),
+ evaluatorIds,
sourceTable,
evalVersion: "v2",
+ ...(evaluatorMappings ? { evaluatorMappings } : {}),
});
} catch {
return;
}
+ capture("batch_eval:run", {
+ evaluatorCount: evaluatorIds.length,
+ mappingOverrideCount,
+ sourceTable,
+ isForceV3Experience: forceV3Experience,
+ isV4: true,
+ });
+
showSuccessToast({
title: "Evaluation queued",
description: isExperimentsSource
- ? `Scheduled evaluation for items from ${displayCount} selected experiment${displayCount === 1 ? "" : "s"} with ${selectedEvaluators.length} ${selectedEvaluators.length === 1 ? "evaluator" : "evaluators"}.`
+ ? `Scheduled evaluation for items from ${displayCount} selected experiment${displayCount === 1 ? "" : "s"} with ${evaluatorIds.length} ${evaluatorIds.length === 1 ? "evaluator" : "evaluators"}.`
: sourceTable === SourceTable.EXPERIMENT_ITEMS
- ? `Scheduled evaluation for up to ${displayCount} experiment item${displayCount === 1 ? "" : "s"} across ${experimentItemsExperimentCount} experiment${experimentItemsExperimentCount === 1 ? "" : "s"} with ${selectedEvaluators.length} ${selectedEvaluators.length === 1 ? "evaluator" : "evaluators"}.`
- : `Scheduled evaluation for ${displayCount} selected ${scopeLabel}${displayCount === 1 ? "" : "s"} with ${selectedEvaluators.length} ${selectedEvaluators.length === 1 ? "evaluator" : "evaluators"}.`,
+ ? `Scheduled evaluation for up to ${displayCount} experiment item${displayCount === 1 ? "" : "s"} across ${experimentItemsExperimentCount} experiment${experimentItemsExperimentCount === 1 ? "" : "s"} with ${evaluatorIds.length} ${evaluatorIds.length === 1 ? "evaluator" : "evaluators"}.`
+ : `Scheduled evaluation for ${displayCount} selected ${scopeLabel}${displayCount === 1 ? "" : "s"} with ${evaluatorIds.length} ${evaluatorIds.length === 1 ? "evaluator" : "evaluators"}.`,
link: {
href: `/project/${projectId}/settings/batch-actions`,
text: "View batch actions",
@@ -189,10 +314,22 @@ export function RunEvaluationDialog(props: RunEvaluationDialogProps) {
props.onClose();
};
+ const createEvaluatorHref = getCreateEvaluatorHref({
+ projectId,
+ forceV3Experience,
+ });
+
return (
<>
<Dialog open onOpenChange={(open) => !open && props.onClose()}>
- <DialogContent className="flex max-h-[62vh] min-h-[38vh] max-w-2xl flex-col">
+ <DialogContent
+ {...(showMappingEditor ? { size: "lg" as const } : {})}
+ className={
+ showMappingEditor
+ ? "flex max-h-[85vh] min-h-[38vh] flex-col"
+ : "flex max-h-[62vh] min-h-[38vh] max-w-2xl flex-col"
+ }
+ >
<DialogHeader>
<DialogTitle>
{isExperimentsSource
@@ -204,40 +341,63 @@ export function RunEvaluationDialog(props: RunEvaluationDialogProps) {
<DialogDescription>
{step === "confirm"
? "Review your evaluation configuration before running."
- : "Select one or more evaluators."}
+ : showMappingEditor
+ ? "Select evaluators and review their variable mappings."
+ : "Select one or more evaluators."}
</DialogDescription>
</DialogHeader>
- <DialogBody className="flex-1 overflow-hidden">
- {step === "select-evaluator" ? (
- <EvaluatorSelectionStep
- eligibleEvaluators={eligibleEvaluators}
- selectedEvaluators={selectedEvaluators}
- isQueryLoading={evaluatorsQuery.isLoading}
- isQueryError={evaluatorsQuery.isError}
- queryErrorMessage={evaluatorsQuery.error?.message}
- previewObservation={previewQuery.data?.[0]}
- isPreviewLoading={previewQuery.isLoading}
- selectedEvaluatorIds={selectedEvaluatorIds}
- evaluatorSearchQuery={evaluatorSearchQuery}
- onSearchQueryChange={setEvaluatorSearchQuery}
- onToggleEvaluator={toggleEvaluatorSelection}
- createEvaluatorHref={getCreateEvaluatorHref({
- projectId,
- forceV3Experience,
- })}
- />
+ <DialogBody
+ className={
+ showMappingEditor
+ ? "min-h-0 flex-1 overflow-y-auto"
+ : "flex-1 overflow-hidden"
+ }
+ >
+ {isExperiencePending ? (
+ <Skeleton className="h-20 w-full" />
+ ) : step === "select-evaluator" ? (
+ showMappingEditor ? (
+ <EvaluatorMappingStep
+ projectId={projectId}
+ store={ruleSetupStore}
+ evaluatorOptions={evaluatorOptions}
+ isQueryLoading={evaluatorsQuery.isLoading}
+ isQueryError={evaluatorsQuery.isError}
+ queryErrorMessage={evaluatorsQuery.error?.message}
+ search={mappingSearch}
+ onSearchChange={(value) => {
+ setMappingSearch(value);
+ debouncedMappingSearch(value);
+ }}
+ sampleObject={sampleObject}
+ costObservationCount={costObservationCount}
+ />
+ ) : (
+ <EvaluatorSelectionStep
+ eligibleEvaluators={eligibleEvaluators}
+ selectedEvaluators={selectedEvaluators}
+ isQueryLoading={evaluatorsQuery.isLoading}
+ isQueryError={evaluatorsQuery.isError}
+ queryErrorMessage={evaluatorsQuery.error?.message}
+ previewObservation={previewQuery.data?.[0]}
+ isPreviewLoading={previewQuery.isLoading}
+ selectedEvaluatorIds={selectedEvaluatorIds}
+ evaluatorSearchQuery={evaluatorSearchQuery}
+ onSearchQueryChange={setEvaluatorSearchQuery}
+ onToggleEvaluator={toggleEvaluatorSelection}
+ />
+ )
) : (
<ConfirmationStep
projectId={projectId}
displayCount={displayCount}
- evaluators={selectedEvaluators.map((e) => ({
- id: e.id,
- name: e.scoreName,
+ evaluators={selectedEvaluators.map((evaluator) => ({
+ id: evaluator.id,
+ name: evaluator.scoreName,
}))}
hideCount={sourceTable !== SourceTable.EVENTS}
sourceTable={sourceTable}
- experimentCount={experimentItemsExperimentCount}
/>
)}
</DialogBody>
@@ -256,27 +416,95 @@ export function RunEvaluationDialog(props: RunEvaluationDialogProps) {
<div />
)}
- {step === "select-evaluator" ? (
- <Button
- onClick={() => setStep("confirm")}
- disabled={selectedEvaluators.length === 0}
- >
- Continue{" "}
- {selectedEvaluators.length > 0
- ? `with ${selectedEvaluators.length} evaluator(s)`
- : null}
- </Button>
- ) : (
- <Button
- onClick={onSubmit}
- loading={runEvaluationMutation.isPending}
- >
- Run Evaluation
- </Button>
- )}
+ <div className="flex items-center gap-2">
+ {step !== "confirm" ? (
+ <CreateEvaluatorButton href={createEvaluatorHref} />
+ ) : null}
+ {showMappingEditor ? (
+ <MappingRunButton
+ disabledReason={mappingRunDisabledReason}
+ selectedCount={selectedCount}
+ loading={runEvaluationMutation.isPending}
+ onClick={onSubmit}
+ />
+ ) : step === "select-evaluator" ? (
+ <Button
+ onClick={() => setStep("confirm")}
+ disabled={isExperiencePending || selectedCount === 0}
+ >
+ Continue{" "}
+ {selectedCount > 0
+ ? `with ${selectedCount} evaluator(s)`
+ : null}
+ </Button>
+ ) : (
+ <Button
+ onClick={onSubmit}
+ loading={runEvaluationMutation.isPending}
+ >
+ Run Evaluation
+ </Button>
+ )}
+ </div>
</DialogFooter>
</DialogContent>
</Dialog>
</>
);
}
+
+function CreateEvaluatorButton({ href }: { href: string }) {
+ return (
+ <Button variant="secondary" className="gap-1.5" asChild>
+ <Link
+ href={href}
+ target="_blank"
+ rel="noreferrer"
+ aria-label="Create new Evaluator (opens in a new tab)"
+ >
+ <Plus className="size-4 shrink-0" aria-hidden="true" />
+ Create new Evaluator
+ <ExternalLink className="size-3.5 shrink-0" aria-hidden="true" />
+ </Link>
+ </Button>
+ );
+}
+
+function MappingRunButton({
+ disabledReason,
+ selectedCount,
+ loading,
+ onClick,
+}: {
+ disabledReason: string | null;
+ selectedCount: number;
+ loading: boolean;
+ onClick: () => void;
+}) {
+ const button = (
+ <Button
+ onClick={onClick}
+ disabled={Boolean(disabledReason)}
+ loading={loading}
+ className={disabledReason ? "pointer-events-none" : undefined}
+ >
+ Run Evaluation
+ {selectedCount > 0 ? ` with ${selectedCount} evaluator(s)` : null}
+ </Button>
+ );
+
+ if (!disabledReason) {
+ return button;
+ }
+
+ return (
+ <Tooltip>
+ <TooltipTrigger asChild>
+ <span className="inline-flex cursor-not-allowed" tabIndex={0}>
+ {button}
+ </span>
+ </TooltipTrigger>
+ <TooltipContent>{disabledReason}</TooltipContent>
+ </Tooltip>
+ );
+}
diff --git a/web/src/features/batch-actions/components/RunEvaluationDialog/utils.clienttest.ts b/web/src/features/batch-actions/components/RunEvaluationDialog/utils.clienttest.ts
index e1b4444e35..1dbaea4208 100644
--- a/web/src/features/batch-actions/components/RunEvaluationDialog/utils.clienttest.ts
+++ b/web/src/features/batch-actions/components/RunEvaluationDialog/utils.clienttest.ts
@@ -1,5 +1,10 @@
import { describe, expect, it } from "vitest";
-import { getCreateEvaluatorHref } from "./utils";
+import { BatchEvalSourceTable, EvalTemplateType } from "@langfuse/shared";
+import {
+ getBatchEvalCostObservationCount,
+ getCreateEvaluatorHref,
+ hasCompleteBatchEvalMappings,
+} from "./utils";
describe("getCreateEvaluatorHref", () => {
it("links forced-v3 projects to the legacy evaluator UI", () => {
@@ -20,3 +25,102 @@ describe("getCreateEvaluatorHref", () => {
).toBe("/project/project-id/evals?gallery=open");
});
});
+
+describe("getBatchEvalCostObservationCount", () => {
+ it("returns the selected observation count for events", () => {
+ expect(
+ getBatchEvalCostObservationCount({
+ displayCount: 12,
+ sourceTable: BatchEvalSourceTable.EVENTS,
+ }),
+ ).toBe(12);
+ });
+
+ it("uses the already-expanded observation count for experiment items", () => {
+ expect(
+ getBatchEvalCostObservationCount({
+ displayCount: 12,
+ sourceTable: BatchEvalSourceTable.EXPERIMENT_ITEMS,
+ }),
+ ).toBe(12);
+ });
+
+ it("returns null when the observation count is unknown", () => {
+ expect(
+ getBatchEvalCostObservationCount({
+ displayCount: 5,
+ sourceTable: BatchEvalSourceTable.EXPERIMENTS,
+ }),
+ ).toBeNull();
+ });
+});
+
+describe("hasCompleteBatchEvalMappings", () => {
+ it("treats code evaluators as complete without a mapping", () => {
+ expect(
+ hasCompleteBatchEvalMappings([
+ {
+ evaluatorType: EvalTemplateType.CODE,
+ variableMapping: null,
+ defaultVariableMapping: [],
+ },
+ ]),
+ ).toBe(true);
+ });
+
+ it("rejects an LLM mapping with an empty source column", () => {
+ expect(
+ hasCompleteBatchEvalMappings([
+ {
+ evaluatorType: EvalTemplateType.LLM_AS_JUDGE,
+ variableMapping: [
+ { templateVariable: "output", selectedColumnId: "" },
+ ],
+ defaultVariableMapping: [
+ { templateVariable: "output", selectedColumnId: "output" },
+ ],
+ },
+ ]),
+ ).toBe(false);
+ });
+
+ it("accepts inherited mappings that already have source columns", () => {
+ expect(
+ hasCompleteBatchEvalMappings([
+ {
+ evaluatorType: EvalTemplateType.LLM_AS_JUDGE,
+ variableMapping: null,
+ defaultVariableMapping: [
+ { templateVariable: "output", selectedColumnId: "output" },
+ ],
+ },
+ ]),
+ ).toBe(true);
+ });
+
+ it("rejects an empty LLM mapping when the prompt still has variables", () => {
+ expect(
+ hasCompleteBatchEvalMappings([
+ {
+ evaluatorType: EvalTemplateType.LLM_AS_JUDGE,
+ variableMapping: null,
+ defaultVariableMapping: [],
+ requiredVariables: ["user_input", "assistant_output"],
+ },
+ ]),
+ ).toBe(false);
+ });
+
+ it("accepts an LLM evaluator whose prompt has no variables", () => {
+ expect(
+ hasCompleteBatchEvalMappings([
+ {
+ evaluatorType: EvalTemplateType.LLM_AS_JUDGE,
+ variableMapping: null,
+ defaultVariableMapping: [],
+ requiredVariables: [],
+ },
+ ]),
+ ).toBe(true);
+ });
+});
diff --git a/web/src/features/batch-actions/components/RunEvaluationDialog/utils.ts b/web/src/features/batch-actions/components/RunEvaluationDialog/utils.ts
index 752cb71829..9dbd2bf9c9 100644
--- a/web/src/features/batch-actions/components/RunEvaluationDialog/utils.ts
+++ b/web/src/features/batch-actions/components/RunEvaluationDialog/utils.ts
@@ -1,4 +1,6 @@
import {
+ BatchEvalSourceTable,
+ EvalTemplateType,
extractValueFromObjectAsString,
zipToolCallsFromRecord,
type BatchActionQuery,
@@ -82,6 +84,53 @@ export function renderPromptPreviewFromObservation(params: {
: renderedPrompt;
}
+/**
+ * Observations this batch will score, or null when the count cannot be known
+ * up front (experiment-scoped runs).
+ *
+ * `displayCount` is already the observation count for experiment items:
+ * callers expand selected rows / select-all totals across experiments.
+ */
+export function getBatchEvalCostObservationCount(params: {
+ displayCount: number;
+ sourceTable: BatchEvalSourceTable;
+}): number | null {
+ const { displayCount, sourceTable } = params;
+ if (sourceTable === BatchEvalSourceTable.EXPERIMENTS) {
+ return null;
+ }
+ return displayCount;
+}
+
+export function hasCompleteBatchEvalMappings(
+ assignments: Array<{
+ evaluatorType: EvalTemplateType;
+ variableMapping: ObservationVariableMapping[] | null;
+ defaultVariableMapping: ObservationVariableMapping[];
+ requiredVariables?: string[];
+ }>,
+): boolean {
+ return assignments.every((assignment) => {
+ if (assignment.evaluatorType === EvalTemplateType.CODE) {
+ return true;
+ }
+ const mapping =
+ assignment.variableMapping ?? assignment.defaultVariableMapping;
+ const mappedVariables = new Set(
+ mapping
+ .filter((entry) => Boolean(entry.selectedColumnId?.trim()))
+ .map((entry) => entry.templateVariable),
+ );
+ const requiredVariables = assignment.requiredVariables ?? [];
+ if (requiredVariables.length > 0) {
+ return requiredVariables.every((variable) =>
+ mappedVariables.has(variable),
+ );
+ }
+ return mapping.every((entry) => Boolean(entry.selectedColumnId?.trim()));
+ });
+}
+
export function buildQueryWithSelectedIds(params: {
query: BatchActionQuery;
selectAll: boolean;
diff --git a/web/src/features/batch-actions/server/prepareBatchEvalEvaluatorMappings.ts b/web/src/features/batch-actions/server/prepareBatchEvalEvaluatorMappings.ts
new file mode 100644
index 0000000000..624f4245d8
--- /dev/null
+++ b/web/src/features/batch-actions/server/prepareBatchEvalEvaluatorMappings.ts
@@ -0,0 +1,91 @@
+import {
+ EvalTemplateType,
+ InvalidRequestError,
+ type BatchEvalEvaluatorMapping,
+} from "@langfuse/shared";
+import type { PrismaClient } from "@langfuse/shared/src/db";
+import { findEvaluatorsByIds } from "@/src/features/evals/v2/server/evaluators/evaluatorRepository";
+import { reconcileEvaluatorPromptMessages } from "@/src/features/evals/v2/server/evaluators/evaluatorService";
+import {
+ assertCompleteEvaluatorVariableMapping,
+ extractEvaluatorPromptVariables,
+} from "@/src/features/evals/v2/server/evaluators/evaluatorValidation";
+import { prepareModernRuleVariableMapping } from "@/src/features/evals/v2/fns/variableMapping/prepareModernRuleVariableMapping";
+
+export async function prepareBatchEvalEvaluatorMappings(params: {
+ prisma: PrismaClient;
+ projectId: string;
+ mappings: BatchEvalEvaluatorMapping[];
+}): Promise<BatchEvalEvaluatorMapping[]> {
+ const { prisma, projectId, mappings } = params;
+ const evaluatorIds = mappings.map(({ evaluatorId }) => evaluatorId);
+ const evaluators = await findEvaluatorsByIds({
+ prisma,
+ projectId,
+ evaluatorIds,
+ });
+ if (evaluators.length !== evaluatorIds.length) {
+ throw new InvalidRequestError(
+ "Selected evaluators are missing or incompatible with batch evaluation.",
+ );
+ }
+
+ const evaluatorById = new Map(
+ evaluators.map((evaluator) => [evaluator.id, evaluator]),
+ );
+
+ return mappings.map((mapping) => {
+ const evaluator = evaluatorById.get(mapping.evaluatorId);
+ if (!evaluator) {
+ throw new InvalidRequestError(
+ "Selected evaluators are missing or incompatible with batch evaluation.",
+ );
+ }
+ try {
+ const latestVersion = evaluator.versions[0];
+ if (!latestVersion) {
+ throw new InvalidRequestError("Evaluator version not found");
+ }
+
+ if (evaluator.type === EvalTemplateType.CODE) {
+ if (mapping.variableMapping !== null) {
+ throw new InvalidRequestError(
+ "Code evaluator mappings are managed by Langfuse and cannot be provided.",
+ );
+ }
+ return {
+ evaluatorId: mapping.evaluatorId,
+ variableMapping: null,
+ };
+ }
+
+ const prepared = prepareModernRuleVariableMapping(
+ latestVersion.variableMapping,
+ evaluator.type,
+ );
+ const storedVariableMapping =
+ mapping.variableMapping ?? prepared.initialVariableMapping;
+ const promptMessages = reconcileEvaluatorPromptMessages({
+ prompt: latestVersion.prompt,
+ promptMessages: latestVersion.promptMessages,
+ });
+ assertCompleteEvaluatorVariableMapping({
+ promptVariables: extractEvaluatorPromptVariables(promptMessages),
+ variableMapping:
+ storedVariableMapping ?? prepared.defaultVariableMapping,
+ });
+
+ return {
+ evaluatorId: mapping.evaluatorId,
+ variableMapping: storedVariableMapping,
+ };
+ } catch (error) {
+ if (error instanceof InvalidRequestError) {
+ throw new InvalidRequestError(
+ `Evaluator "${evaluator.name}": ${error.message}`,
+ );
+ }
+ throw error;
+ }
+ });
+}
diff --git a/web/src/features/batch-actions/server/runEvaluationRouter.ts b/web/src/features/batch-actions/server/runEvaluationRouter.ts
index d3578ee531..5e860abcc8 100644
--- a/web/src/features/batch-actions/server/runEvaluationRouter.ts
+++ b/web/src/features/batch-actions/server/runEvaluationRouter.ts
@@ -23,6 +23,7 @@ import {
import { env } from "@/src/env.mjs";
import { CreateObservationBatchEvaluationActionSchema } from "../validation";
import { batchEligibleEvaluatorWhere } from "@/src/features/evals/v2/server/evaluators/evaluatorRepository";
+import { prepareBatchEvalEvaluatorMappings } from "./prepareBatchEvalEvaluatorMappings";
export const runEvaluationRouter = createTRPCRouter({
create: protectedProjectProcedure
@@ -40,6 +41,7 @@ export const runEvaluationRouter = createTRPCRouter({
query,
evaluatorIds: rawEvaluatorIds,
sourceTable = BatchEvalSourceTable.EVENTS,
+ evaluatorMappings: rawEvaluatorMappings,
} = input;
if (env.LANGFUSE_MIGRATION_V4_ALLOW_PREVIEW_OPT_IN !== "true") {
@@ -111,6 +113,15 @@ export const runEvaluationRouter = createTRPCRouter({
});
}
+ const evaluatorMappings =
+ input.evalVersion === "v2" && rawEvaluatorMappings
+ ? await prepareBatchEvalEvaluatorMappings({
+ prisma: ctx.prisma,
+ projectId,
+ mappings: rawEvaluatorMappings,
+ })
+ : undefined;
+
// Event comments live in Postgres, so resolve them for the preflight
// count while retaining the original query for the queued worker.
const commentFilterResult =
@@ -145,6 +156,7 @@ export const runEvaluationRouter = createTRPCRouter({
const batchConfig = {
evaluatorIds,
...(input.evalVersion ? { evalVersion: input.evalVersion } : {}),
+ ...(evaluatorMappings ? { evaluatorMappings } : {}),
};
logger.info(
@@ -193,6 +205,7 @@ export const runEvaluationRouter = createTRPCRouter({
...(batchConfig.evalVersion
? { evalVersion: batchConfig.evalVersion }
: {}),
+ ...(evaluatorMappings ? { evaluatorMappings } : {}),
},
},
{
diff --git a/web/src/features/batch-actions/validation.ts b/web/src/features/batch-actions/validation.ts
index 23d9d81ad3..96f8ce1b14 100644
--- a/web/src/features/batch-actions/validation.ts
+++ b/web/src/features/batch-actions/validation.ts
@@ -2,22 +2,63 @@ import z from "zod";
import {
ObservationAddToDatasetConfigSchema,
BatchActionQuerySchema,
+ BatchEvalEvaluatorMappingSchema,
BatchEvalSourceTableSchema,
} from "@langfuse/shared";
+/** Matches the evaluator options page size used by the run-evaluation dialog. */
+export const BATCH_EVAL_EVALUATOR_LIMIT = 100;
+
export const CreateObservationAddToDatasetActionSchema = z.object({
projectId: z.string(),
query: BatchActionQuerySchema,
config: ObservationAddToDatasetConfigSchema,
});
-export const CreateObservationBatchEvaluationActionSchema = z.object({
- projectId: z.string(),
- query: BatchActionQuerySchema,
- evaluatorIds: z.array(z.string()).min(1),
- sourceTable: BatchEvalSourceTableSchema.default("events"),
- evalVersion: z.literal("v2").optional(),
-});
+export const CreateObservationBatchEvaluationActionSchema = z
+ .object({
+ projectId: z.string(),
+ query: BatchActionQuerySchema,
+ evaluatorIds: z.array(z.string()).min(1).max(BATCH_EVAL_EVALUATOR_LIMIT),
+ sourceTable: BatchEvalSourceTableSchema.default("events"),
+ evalVersion: z.literal("v2").optional(),
+ evaluatorMappings: z
+ .array(BatchEvalEvaluatorMappingSchema)
+ .max(BATCH_EVAL_EVALUATOR_LIMIT)
+ .optional(),
+ })
+ .superRefine((value, ctx) => {
+ if (!value.evaluatorMappings) return;
+
+ if (value.evalVersion !== "v2") {
+ ctx.addIssue({
+ code: "custom",
+ message: "Variable mapping overrides require evaluator v2.",
+ path: ["evaluatorMappings"],
+ });
+ return;
+ }
+
+ const evaluatorIds = new Set(value.evaluatorIds);
+ const mappingIds = new Set<string>();
+ for (const [index, mapping] of value.evaluatorMappings.entries()) {
+ if (!evaluatorIds.has(mapping.evaluatorId)) {
+ ctx.addIssue({
+ code: "custom",
+ message: "Mapping override must refer to a selected evaluator.",
+ path: ["evaluatorMappings", index, "evaluatorId"],
+ });
+ }
+ if (mappingIds.has(mapping.evaluatorId)) {
+ ctx.addIssue({
+ code: "custom",
+ message: "An evaluator can only have one mapping override.",
+ path: ["evaluatorMappings", index, "evaluatorId"],
+ });
+ }
+ mappingIds.add(mapping.evaluatorId);
+ }
+ });
export const GetBatchActionByIdSchema = z.object({
projectId: z.string(),
diff --git a/web/src/features/evals/v2/components/Rules/EvaluatorAssignmentsEditor/EvaluatorAssignmentsEditor.tsx b/web/src/features/evals/v2/components/Rules/EvaluatorAssignmentsEditor/EvaluatorAssignmentsEditor.tsx
index d7200c5765..d232d9ab89 100644
--- a/web/src/features/evals/v2/components/Rules/EvaluatorAssignmentsEditor/EvaluatorAssignmentsEditor.tsx
+++ b/web/src/features/evals/v2/components/Rules/EvaluatorAssignmentsEditor/EvaluatorAssignmentsEditor.tsx
@@ -97,9 +97,9 @@ export function EvaluatorAssignmentsEditor({
variant="ghost"
size="sm"
disabled={disabled}
- className="text-foreground hover:text-foreground h-auto px-0 py-0 text-xs underline-offset-4 hover:bg-transparent hover:underline"
+ className="text-foreground hover:text-foreground inline-flex h-auto items-center gap-1.5 px-0 py-0 text-xs leading-none underline-offset-4 hover:bg-transparent hover:underline"
>
- <Plus className="mr-1.5 h-3.5 w-3.5" />
+ <Plus className="size-3.5 shrink-0" aria-hidden="true" />
Attach another evaluator
</Button>
)}
@@ -146,6 +146,7 @@ export function EvaluatorAssignmentsEditor({
defaultVariableMapping:
evaluator.defaultVariableMapping,
variableMapping: evaluator.initialVariableMapping,
+ requiredVariables: evaluator.requiredVariables,
});
setPickerOpen(false);
}}
diff --git a/web/src/features/evals/v2/components/Rules/RuleSetup/components/RuleEvaluatorCostEstimate.tsx b/web/src/features/evals/v2/components/Rules/RuleSetup/components/RuleEvaluatorCostEstimate.tsx
index 7197a8a231..734581266a 100644
--- a/web/src/features/evals/v2/components/Rules/RuleSetup/components/RuleEvaluatorCostEstimate.tsx
+++ b/web/src/features/evals/v2/components/Rules/RuleSetup/components/RuleEvaluatorCostEstimate.tsx
@@ -18,7 +18,9 @@ export function RuleEvaluatorCostEstimate({
<span className="cursor-help font-mono text-sm tabular-nums underline decoration-dotted underline-offset-4">
{estimate.estimatedCostUsd === null
? "Unavailable"
- : `≈ ${usdFormatter(estimate.estimatedCostUsd, 2, 2)} / week`}
+ : estimate.period === "selection"
+ ? `≈ ${usdFormatter(estimate.estimatedCostUsd, 2, 2)}`
+ : `≈ ${usdFormatter(estimate.estimatedCostUsd, 2, 2)} / week`}
</span>
</TooltipTrigger>
<TooltipContent className="max-w-72">
diff --git a/web/src/features/evals/v2/fns/formatEvaluatorCostCalculation.clienttest.ts b/web/src/features/evals/v2/fns/formatEvaluatorCostCalculation.clienttest.ts
index 0ff266eedf..8d39a5b9eb 100644
--- a/web/src/features/evals/v2/fns/formatEvaluatorCostCalculation.clienttest.ts
+++ b/web/src/features/evals/v2/fns/formatEvaluatorCostCalculation.clienttest.ts
@@ -59,4 +59,34 @@ describe("formatEvaluatorCostCalculation", () => {
"Code evaluators do not call an LLM, so they do not incur model-provider / LLM costs.",
);
});
+
+ it("does not claim a 7-day window when a selection has no priced test", () => {
+ expect(
+ formatEvaluatorCostCalculation({
+ matchingObservations: 3,
+ sampling: 1,
+ testRunCostUsd: null,
+ estimatedCostUsd: null,
+ evaluatorType: EvalTemplateTypeEnum.LLM_AS_JUDGE,
+ period: "selection",
+ }),
+ ).toBe(
+ "No cost-bearing evaluator trace was available, and the fallback test call did not return a usable model cost. Expected cost would be charged to your linked API key, not Langfuse.",
+ );
+ });
+
+ it("describes a one-shot selection instead of a weekly total", () => {
+ expect(
+ formatEvaluatorCostCalculation({
+ matchingObservations: 12,
+ sampling: 1,
+ testRunCostUsd: 0.01,
+ estimatedCostUsd: 0.12,
+ evaluatorType: EvalTemplateTypeEnum.LLM_AS_JUDGE,
+ period: "selection",
+ }),
+ ).toBe(
+ "12 observations × 100% sampling × $0.01 per evaluation = ≈ $0.12 for this run. Expected cost on your linked API key (not Langfuse). Per-evaluation cost uses the latest cost-bearing evaluator trace from that period, or a fallback test call.",
+ );
+ });
});
diff --git a/web/src/features/evals/v2/fns/formatEvaluatorCostCalculation.ts b/web/src/features/evals/v2/fns/formatEvaluatorCostCalculation.ts
index a154c7d077..6a4f8faa19 100644
--- a/web/src/features/evals/v2/fns/formatEvaluatorCostCalculation.ts
+++ b/web/src/features/evals/v2/fns/formatEvaluatorCostCalculation.ts
@@ -7,28 +7,36 @@ export function formatEvaluatorCostCalculation({
testRunCostUsd,
estimatedCostUsd,
evaluatorType,
+ period = "week",
}: {
matchingObservations: number;
sampling: number;
testRunCostUsd: number | null;
estimatedCostUsd: number | null;
evaluatorType: EvalTemplateType;
+ period?: "week" | "selection";
}) {
if (evaluatorType === EvalTemplateTypeEnum.CODE) {
return "Code evaluators do not call an LLM, so they do not incur model-provider / LLM costs.";
}
- const observations = `${numberFormatter(matchingObservations, 0)} matching observations`;
+ const isSelection = period === "selection";
+ const observations = isSelection
+ ? `${numberFormatter(matchingObservations, 0)} observations`
+ : `${numberFormatter(matchingObservations, 0)} matching observations`;
const samplingRate = `${numberFormatter(sampling * 100, 0, 2)}% sampling`;
- const scope =
- "Expected cost on your linked API key (not Langfuse). Matching observations are based on the last 7 days.";
+ const periodLabel = isSelection ? "for this run" : "/ week";
+ const scope = isSelection
+ ? "Expected cost on your linked API key (not Langfuse)."
+ : "Expected cost on your linked API key (not Langfuse). Matching observations are based on the last 7 days.";
if (matchingObservations === 0) {
- return `${observations} × ${samplingRate} = ≈ ${usdFormatter(0, 2, 2)} / week. ${scope}`;
+ return `${observations} × ${samplingRate} = ≈ ${usdFormatter(0, 2, 2)} ${periodLabel}. ${scope}`;
}
if (testRunCostUsd === null || estimatedCostUsd === null) {
- return `No cost-bearing evaluator trace was available from the last 7 days, and the fallback test call did not return a usable model cost. Expected cost would be charged to your linked API key, not Langfuse.`;
+ const window = isSelection ? "" : " from the last 7 days";
+ return `No cost-bearing evaluator trace was available${window}, and the fallback test call did not return a usable model cost. Expected cost would be charged to your linked API key, not Langfuse.`;
}
- return `${observations} × ${samplingRate} × ${usdFormatter(testRunCostUsd, 2, 6)} per evaluation = ≈ ${usdFormatter(estimatedCostUsd, 2, 2)} / week. ${scope} Per-evaluation cost uses the latest cost-bearing evaluator trace from that period, or a fallback test call.`;
+ return `${observations} × ${samplingRate} × ${usdFormatter(testRunCostUsd, 2, 6)} per evaluation = ≈ ${usdFormatter(estimatedCostUsd, 2, 2)} ${periodLabel}. ${scope} Per-evaluation cost uses the latest cost-bearing evaluator trace from that period, or a fallback test call.`;
}
diff --git a/web/src/features/evals/v2/fns/variableMapping/coverEvaluatorPromptVariables.ts b/web/src/features/evals/v2/fns/variableMapping/coverEvaluatorPromptVariables.ts
new file mode 100644
index 0000000000..784bc59fc8
--- /dev/null
+++ b/web/src/features/evals/v2/fns/variableMapping/coverEvaluatorPromptVariables.ts
@@ -0,0 +1,25 @@
+import type { ObservationVariableMapping } from "@langfuse/shared";
+
+/**
+ * Adds empty mapping rows for prompt variables the stored mapping does not
+ * already list, so the editor can collect a source column for each one.
+ */
+export function coverEvaluatorPromptVariables(
+ mapping: ObservationVariableMapping[],
+ promptVariables: string[],
+): ObservationVariableMapping[] {
+ const existing = new Set(mapping.map((entry) => entry.templateVariable));
+ const missing = promptVariables.filter((variable) => !existing.has(variable));
+ if (missing.length === 0) {
+ return mapping;
+ }
+
+ return [
+ ...mapping,
+ ...missing.map((templateVariable) => ({
+ templateVariable,
+ selectedColumnId: "",
+ jsonSelector: null,
+ })),
+ ];
+}
diff --git a/web/src/features/evals/v2/fns/variableMapping/prepareModernRuleVariableMapping.clienttest.ts b/web/src/features/evals/v2/fns/variableMapping/prepareModernRuleVariableMapping.clienttest.ts
index dddeaa6541..f8644f164d 100644
--- a/web/src/features/evals/v2/fns/variableMapping/prepareModernRuleVariableMapping.clienttest.ts
+++ b/web/src/features/evals/v2/fns/variableMapping/prepareModernRuleVariableMapping.clienttest.ts
@@ -3,6 +3,7 @@ import {
EvalTemplateType,
} from "@langfuse/shared";
import { describe, expect, it } from "vitest";
+import { coverEvaluatorPromptVariables } from "./coverEvaluatorPromptVariables";
import { prepareModernRuleVariableMapping } from "./prepareModernRuleVariableMapping";
describe("prepareModernRuleVariableMapping", () => {
@@ -100,3 +101,25 @@ describe("prepareModernRuleVariableMapping", () => {
});
});
});
+
+describe("coverEvaluatorPromptVariables", () => {
+ it("adds empty rows for prompt variables missing from the stored mapping", () => {
+ expect(
+ coverEvaluatorPromptVariables(
+ [{ templateVariable: "output", selectedColumnId: "output" }],
+ ["input", "output"],
+ ),
+ ).toEqual([
+ { templateVariable: "output", selectedColumnId: "output" },
+ { templateVariable: "input", selectedColumnId: "", jsonSelector: null },
+ ]);
+ });
+
+ it("returns the original mapping when every prompt variable is already listed", () => {
+ const mapping = [
+ { templateVariable: "output", selectedColumnId: "output" },
+ ];
+
+ expect(coverEvaluatorPromptVariables(mapping, ["output"])).toBe(mapping);
+ });
+});
diff --git a/web/src/features/evals/v2/hooks/useRuleCostEstimate.ts b/web/src/features/evals/v2/hooks/useRuleCostEstimate.ts
index 8193fdf264..a60d94eb7f 100644
--- a/web/src/features/evals/v2/hooks/useRuleCostEstimate.ts
+++ b/web/src/features/evals/v2/hooks/useRuleCostEstimate.ts
@@ -14,6 +14,7 @@ export type RuleCostEstimate = {
testRunCostUsd: number | null;
estimatedCostUsd: number | null;
evaluatorType: EvalTemplateType;
+ period?: "week" | "selection";
};
const ESTIMATE_DEBOUNCE_MS = 500;
@@ -27,9 +28,11 @@ type EstimateState = {
export function useRuleCostEstimate({
projectId,
store,
+ matchingObservations: matchingObservationsOverride,
}: {
projectId: string;
store: RuleSetupStore;
+ matchingObservations?: number;
}) {
const { filter, sampling, assignments } = useStore(
store,
@@ -163,7 +166,7 @@ export function useRuleCostEstimate({
);
updateState({
status: "idle",
- estimates: assignments.flatMap((assignment) => {
+ estimates: assignments.flatMap((assignment): RuleCostEstimate[] => {
if (assignment.evaluatorType === EvalTemplateType.CODE) {
return codeEstimates.filter(
({ evaluatorId }) => evaluatorId === assignment.evaluatorId,
@@ -192,17 +195,24 @@ export function useRuleCostEstimate({
return {
...state,
- estimates: state.estimates.map((estimate) => ({
- ...estimate,
- sampling,
- estimatedCostUsd:
- estimate.matchingObservations === 0
- ? 0
- : estimate.testRunCostUsd === null
- ? null
- : estimate.matchingObservations *
- sampling *
- estimate.testRunCostUsd,
- })),
+ estimates: state.estimates.map((estimate) => {
+ const matchingObservations =
+ matchingObservationsOverride ?? estimate.matchingObservations;
+ return {
+ ...estimate,
+ matchingObservations,
+ sampling,
+ period:
+ matchingObservationsOverride === undefined
+ ? ("week" as const)
+ : ("selection" as const),
+ estimatedCostUsd:
+ matchingObservations === 0
+ ? 0
+ : estimate.testRunCostUsd === null
+ ? null
+ : matchingObservations * sampling * estimate.testRunCostUsd,
+ };
+ }),
};
}
diff --git a/web/src/features/evals/v2/types/rules.ts b/web/src/features/evals/v2/types/rules.ts
index e107e12180..abc56cf28e 100644
--- a/web/src/features/evals/v2/types/rules.ts
+++ b/web/src/features/evals/v2/types/rules.ts
@@ -14,6 +14,7 @@ type RuleDraftAssignment = {
evaluatorType: EvalTemplateType;
defaultVariableMapping: ObservationVariableMapping[];
variableMapping: ObservationVariableMapping[] | null;
+ requiredVariables?: string[];
};
export type RuleDraft = {
@@ -31,6 +32,7 @@ export type RuleEvaluatorOption = {
createdByUser?: { name: string | null; email: string | null } | null;
defaultVariableMapping: ObservationVariableMapping[];
initialVariableMapping: ObservationVariableMapping[] | null;
+ requiredVariables?: string[];
};
export type ActivationConfirmationRequest = {
diff --git a/web/src/features/posthog-analytics/usePostHogClientCapture.ts b/web/src/features/posthog-analytics/usePostHogClientCapture.ts
index 16e9a424f0..8e24764d6a 100644
--- a/web/src/features/posthog-analytics/usePostHogClientCapture.ts
+++ b/web/src/features/posthog-analytics/usePostHogClientCapture.ts
@@ -196,6 +196,9 @@ const events = {
"detach_evaluator",
"filter_reused",
],
+ // One-shot batch evaluation from the events / experiments tables.
+ // Counts and enums only — never mapping contents or observation payloads.
+ batch_eval: ["run"],
integrations: [
"posthog_form_submitted",
"blob_storage_form_submitted",
diff --git a/worker/src/__tests__/handleBatchActionJob-comment-filters.unit.test.ts b/worker/src/__tests__/handleBatchActionJob-comment-filters.unit.test.ts
index e92cefddf9..97fdaf8c4b 100644
--- a/worker/src/__tests__/handleBatchActionJob-comment-filters.unit.test.ts
+++ b/worker/src/__tests__/handleBatchActionJob-comment-filters.unit.test.ts
@@ -242,4 +242,47 @@ describe("event batch-action comment filter wiring", () => {
}),
);
});
+
+ it("applies mapping overrides to ruleless in-memory assignments", async () => {
+ const mapping = [{ templateVariable: "output", selectedColumnId: "input" }];
+ mocks.findEvaluators.mockResolvedValue([
+ {
+ id: "standalone-evaluator",
+ name: "Standalone evaluator",
+ projectId: "project-1",
+ type: "LLM_AS_JUDGE",
+ blockedAt: null,
+ versions: [{ id: "version-2", variableMapping: [] }],
+ assignments: [],
+ },
+ ]);
+
+ await runBatchAction({
+ ...createPayload("observation-run-batched-evaluation"),
+ evaluatorIds: ["standalone-evaluator"],
+ evalVersion: "v2",
+ evaluatorMappings: [
+ {
+ evaluatorId: "standalone-evaluator",
+ variableMapping: mapping,
+ },
+ ],
+ });
+
+ expect(mocks.processBatchedObservationEval).toHaveBeenCalledWith(
+ expect.objectContaining({
+ evaluators: [
+ expect.objectContaining({
+ ruleId: null,
+ assignments: [
+ expect.objectContaining({
+ evaluatorId: "standalone-evaluator",
+ variableMapping: mapping,
+ }),
+ ],
+ }),
+ ],
+ }),
+ );
+ });
});
diff --git a/worker/src/features/batchAction/handleBatchActionJob.ts b/worker/src/features/batchAction/handleBatchActionJob.ts
index addc66d4b2..317a45ccc8 100644
--- a/worker/src/features/batchAction/handleBatchActionJob.ts
+++ b/worker/src/features/batchAction/handleBatchActionJob.ts
@@ -476,6 +476,7 @@ export const handleBatchActionJob = async (
evaluatorIds,
batchActionId,
evalVersion,
+ evaluatorMappings,
} = batchActionEvent;
if (!batchActionId) {
@@ -514,6 +515,12 @@ export const handleBatchActionJob = async (
});
evaluatorLabels = stableEvaluators.map(({ name }) => name);
+ const mappingByEvaluatorId = new Map(
+ (evaluatorMappings ?? []).map((mapping) => [
+ mapping.evaluatorId,
+ mapping.variableMapping,
+ ]),
+ );
// A batch run addresses the evaluator directly (`ruleId` stays null),
// but uses a deterministic associated rule as its legacy execution
// anchor so existing readers can still find it.
@@ -531,7 +538,7 @@ export const handleBatchActionJob = async (
{
id: evaluator.id,
evaluatorId: evaluator.id,
- variableMapping: null,
+ variableMapping: mappingByEvaluatorId.get(evaluator.id) ?? null,
evaluator: {
id: evaluator.id,
projectId: evaluator.projectId,
diff --git a/worker/src/features/batchAction/processBatchedObservationEval.test.ts b/worker/src/features/batchAction/processBatchedObservationEval.test.ts
index 9746a744bd..790d4a5929 100644
--- a/worker/src/features/batchAction/processBatchedObservationEval.test.ts
+++ b/worker/src/features/batchAction/processBatchedObservationEval.test.ts
@@ -84,6 +84,7 @@ describe("processBatchedObservationEval", () => {
expect.objectContaining({
configs: evaluators,
executionMode: "MANUAL",
+ executionScopeId: batchActionId,
}),
);
expect(
@@ -150,6 +151,7 @@ describe("processBatchedObservationEval", () => {
expect.objectContaining({
configs: evaluators,
executionMode: "MANUAL",
+ executionScopeId: "batch-action-v2",
}),
);
});
diff --git a/worker/src/features/batchAction/processBatchedObservationEval.ts b/worker/src/features/batchAction/processBatchedObservationEval.ts
index 390d776110..791f31e2ec 100644
--- a/worker/src/features/batchAction/processBatchedObservationEval.ts
+++ b/worker/src/features/batchAction/processBatchedObservationEval.ts
@@ -63,6 +63,7 @@ export async function processBatchedObservationEval(params: {
configs: evaluators,
schedulerDeps,
executionMode: "MANUAL",
+ executionScopeId: batchActionId,
});
}),
),
diff --git a/worker/src/features/evaluation/observationEval/__tests__/createSchedulerDeps.test.ts b/worker/src/features/evaluation/observationEval/__tests__/createSchedulerDeps.test.ts
index 7975f0a593..1dc0b4cbf8 100644
--- a/worker/src/features/evaluation/observationEval/__tests__/createSchedulerDeps.test.ts
+++ b/worker/src/features/evaluation/observationEval/__tests__/createSchedulerDeps.test.ts
@@ -78,4 +78,33 @@ describe("createObservationEvalSchedulerDeps", () => {
);
expect(getLLMQueueInstance).not.toHaveBeenCalled();
});
+
+ it("includes a mapping override on the observation eval payload", async () => {
+ const { createObservationEvalSchedulerDeps } =
+ await import("../createSchedulerDeps");
+ const variableMapping = [
+ { templateVariable: "output", selectedColumnId: "input" },
+ ];
+
+ await createObservationEvalSchedulerDeps().enqueueEvalJob({
+ projectId: "project-1",
+ jobExecutionId: "job-3",
+ observationS3Path: "evals/project-1/observations/obs-1.json",
+ delay: 0,
+ evalTemplateType: EvalTemplateType.LLM_AS_JUDGE,
+ evaluatorId: "evaluator-1",
+ variableMapping,
+ });
+
+ expect(addToLLMQueue).toHaveBeenCalledWith(
+ "llm-as-a-judge-execution-queue",
+ expect.objectContaining({
+ payload: expect.objectContaining({
+ evaluatorId: "evaluator-1",
+ variableMapping,
+ }),
+ }),
+ { delay: 0 },
+ );
+ });
});
diff --git a/worker/src/features/evaluation/observationEval/__tests__/observationEvalProcessor.test.ts b/worker/src/features/evaluation/observationEval/__tests__/observationEvalProcessor.test.ts
index 5136d8d892..f6070cbad3 100644
--- a/worker/src/features/evaluation/observationEval/__tests__/observationEvalProcessor.test.ts
+++ b/worker/src/features/evaluation/observationEval/__tests__/observationEvalProcessor.test.ts
@@ -508,6 +508,44 @@ describe("processObservationEval", () => {
);
});
+ it("uses a queue mapping override for a batch-run evaluator without a rule", async () => {
+ const variableMapping = [
+ { templateVariable: "output", selectedColumnId: "input" },
+ ];
+ (prisma.jobExecution.findFirst as Mock).mockResolvedValue(
+ createMockJobExecution({
+ id: jobExecutionId,
+ projectId,
+ jobConfigurationId: evaluator.id,
+ jobTemplateId: null,
+ }),
+ );
+ (prisma.evaluator.findFirst as Mock).mockResolvedValue(evaluator);
+
+ await processObservationEval({
+ event: {
+ ...baseEvent,
+ executionMode: "MANUAL",
+ evaluatorId: evaluator.id,
+ variableMapping,
+ },
+ executionType: EvalTemplateType.LLM_AS_JUDGE,
+ deps: createMockProcessorDeps(),
+ });
+
+ expect(
+ prisma.evaluationRuleEvaluatorAssignment.findFirst,
+ ).not.toHaveBeenCalled();
+ expect(runLLMAsJudgeEvaluation).toHaveBeenCalledWith(
+ expect.objectContaining({
+ config: expect.objectContaining({ variableMapping }),
+ executionMetadata: expect.not.objectContaining({
+ evaluation_rule_assignment_id: expect.anything(),
+ }),
+ }),
+ );
+ });
+
it("uses an empty variable mapping when neither v2 mapping is configured", async () => {
setupV2Job();
(
diff --git a/worker/src/features/evaluation/observationEval/__tests__/scheduleObservationEvals.test.ts b/worker/src/features/evaluation/observationEval/__tests__/scheduleObservationEvals.test.ts
index b9a642248b..9c4f652867 100644
--- a/worker/src/features/evaluation/observationEval/__tests__/scheduleObservationEvals.test.ts
+++ b/worker/src/features/evaluation/observationEval/__tests__/scheduleObservationEvals.test.ts
@@ -637,6 +637,173 @@ describe("scheduleObservationEvals", () => {
expect(payload).not.toHaveProperty("evaluationRuleId");
});
+ it("should carry a mapping override for ruleless batch runs", async () => {
+ const schedulerDeps = createMockSchedulerDeps();
+ const variableMapping = [
+ { templateVariable: "output", selectedColumnId: "input" },
+ ];
+
+ await scheduleObservationEvals({
+ observation: createMockObservation(),
+ configs: [
+ createMockRule({
+ id: "evaluator-1",
+ ruleId: null,
+ assignments: [
+ {
+ id: "evaluator-1",
+ evaluatorId: "evaluator-1",
+ variableMapping,
+ evaluator: {
+ id: "evaluator-1",
+ projectId: "project-789",
+ type: EvalTemplateType.LLM_AS_JUDGE,
+ },
+ },
+ ],
+ }),
+ ],
+ schedulerDeps,
+ executionMode: "MANUAL",
+ });
+
+ expect(schedulerDeps.enqueueEvalJob).toHaveBeenCalledWith(
+ expect.objectContaining({
+ evaluatorId: "evaluator-1",
+ variableMapping,
+ }),
+ );
+ });
+
+ it("should include an execution scope in the job identity", async () => {
+ const schedulerDeps = createMockSchedulerDeps();
+ const observation = createMockObservation();
+ const rule = createMockRule({
+ id: "evaluator-1",
+ ruleId: null,
+ assignments: [
+ {
+ id: "evaluator-1",
+ evaluatorId: "evaluator-1",
+ variableMapping: null,
+ evaluator: {
+ id: "evaluator-1",
+ projectId: "project-789",
+ type: EvalTemplateType.LLM_AS_JUDGE,
+ },
+ },
+ ],
+ });
+
+ await scheduleObservationEvals({
+ observation,
+ configs: [rule],
+ schedulerDeps,
+ executionMode: "MANUAL",
+ executionScopeId: "batch-action-1",
+ });
+
+ const scopedJobExecutionId = createW3CTraceId(
+ JSON.stringify([
+ "observation-eval",
+ "evaluator-1",
+ "evaluator-1",
+ observation.trace_id,
+ observation.span_id,
+ "batch-action-1",
+ ]),
+ );
+ const unscopedJobExecutionId = createW3CTraceId(
+ JSON.stringify([
+ "observation-eval",
+ "evaluator-1",
+ "evaluator-1",
+ observation.trace_id,
+ observation.span_id,
+ ]),
+ );
+
+ expect(scopedJobExecutionId).not.toBe(unscopedJobExecutionId);
+ expect(schedulerDeps.upsertJobExecution).toHaveBeenCalledWith(
+ expect.objectContaining({ id: scopedJobExecutionId }),
+ );
+ });
+
+ it("should keep live eval job identity unchanged when no execution scope is passed", async () => {
+ const schedulerDeps = createMockSchedulerDeps();
+ const observation = createMockObservation();
+ const rule = createMockRule();
+
+ await scheduleObservationEvals({
+ observation,
+ configs: [rule],
+ schedulerDeps,
+ });
+
+ expect(schedulerDeps.upsertJobExecution).toHaveBeenCalledWith(
+ expect.objectContaining({
+ id: createW3CTraceId(
+ JSON.stringify([
+ "observation-eval",
+ rule.id,
+ "assignment-1",
+ observation.trace_id,
+ observation.span_id,
+ ]),
+ ),
+ }),
+ );
+ });
+
+ it("should omit mapping when a ruleless batch run inherits the version mapping", async () => {
+ const schedulerDeps = createMockSchedulerDeps();
+
+ await scheduleObservationEvals({
+ observation: createMockObservation(),
+ configs: [createMockRule({ id: "evaluator-1", ruleId: null })],
+ schedulerDeps,
+ executionMode: "MANUAL",
+ });
+
+ const payload = vi.mocked(schedulerDeps.enqueueEvalJob).mock.calls[0]![0];
+ expect(payload).toMatchObject({ evaluatorId: "evaluator-1" });
+ expect(payload).not.toHaveProperty("variableMapping");
+ });
+
+ it("should omit mapping for rule-backed assignments", async () => {
+ const schedulerDeps = createMockSchedulerDeps();
+
+ await scheduleObservationEvals({
+ observation: createMockObservation(),
+ configs: [
+ createMockRule({
+ assignments: [
+ {
+ id: "assignment-1",
+ evaluatorId: "evaluator-1",
+ variableMapping: [
+ { templateVariable: "output", selectedColumnId: "input" },
+ ],
+ evaluator: {
+ id: "evaluator-1",
+ projectId: "project-789",
+ type: EvalTemplateType.LLM_AS_JUDGE,
+ },
+ },
+ ],
+ }),
+ ],
+ schedulerDeps,
+ });
+
+ const payload = vi.mocked(schedulerDeps.enqueueEvalJob).mock.calls[0]![0];
+ expect(payload).toMatchObject({
+ evaluatorId: "evaluator-1",
+ evaluationRuleId: "rule-1",
+ });
+ expect(payload).not.toHaveProperty("variableMapping");
+ });
+
it("should not carry evaluator identity for legacy configs", async () => {
const schedulerDeps = createMockSchedulerDeps();
diff --git a/worker/src/features/evaluation/observationEval/createSchedulerDeps.ts b/worker/src/features/evaluation/observationEval/createSchedulerDeps.ts
index c4b0c1d923..c9f3904629 100644
--- a/worker/src/features/evaluation/observationEval/createSchedulerDeps.ts
+++ b/worker/src/features/evaluation/observationEval/createSchedulerDeps.ts
@@ -82,6 +82,9 @@ export function createObservationEvalSchedulerDeps(): ObservationEvalSchedulerDe
...(params.evaluationRuleId
? { evaluationRuleId: params.evaluationRuleId }
: {}),
+ ...(params.variableMapping != null
+ ? { variableMapping: params.variableMapping }
+ : {}),
};
if (params.evalTemplateType === EvalTemplateType.CODE) {
diff --git a/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts b/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts
index 81908385b0..6e8ac66e9f 100644
--- a/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts
+++ b/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts
@@ -358,9 +358,16 @@ async function resolveObservationEvalExecution(params: {
where: { id: evaluatorId, projectId, type: executionType },
include: evaluatorInclude,
});
- return evaluator
- ? buildV2Execution({ rule: null, assignment: null, evaluator })
- : { type: "cancelled" as const, reason: "evaluator-unavailable" };
+ if (!evaluator) {
+ return { type: "cancelled" as const, reason: "evaluator-unavailable" };
+ }
+ // Ruleless batch runs have no assignment row. A mapping on the queue
+ // payload is the override; omitting it inherits the version mapping.
+ const assignment =
+ event.variableMapping !== undefined
+ ? { variableMapping: event.variableMapping }
+ : null;
+ return buildV2Execution({ rule: null, assignment, evaluator });
}
// The assignment is what authorizes this execution — it is the (rule,
@@ -460,10 +467,10 @@ type ResolvedEvaluationRule = Prisma.EvaluationRuleGetPayload<object>;
*/
function buildV2Execution(params: {
rule: ResolvedEvaluationRule | null;
- assignment: Pick<
- Prisma.EvaluationRuleEvaluatorAssignmentGetPayload<object>,
- "id" | "variableMapping"
- > | null;
+ assignment: {
+ id?: string;
+ variableMapping: Prisma.JsonValue;
+ } | null;
evaluator: ResolvedEvaluator;
}) {
const { rule, assignment, evaluator } = params;
diff --git a/worker/src/features/evaluation/observationEval/scheduleObservationEvals.ts b/worker/src/features/evaluation/observationEval/scheduleObservationEvals.ts
index a3c8f025cc..8ec12af32e 100644
--- a/worker/src/features/evaluation/observationEval/scheduleObservationEvals.ts
+++ b/worker/src/features/evaluation/observationEval/scheduleObservationEvals.ts
@@ -1,3 +1,4 @@
+import type { ObservationVariableMapping } from "@langfuse/shared";
import {
type ObservationForEval,
type ObservationEvalAssignment,
@@ -19,6 +20,7 @@ import {
type EvalExecutionMode,
canRunEvalRule,
mapEventEvalFilterColumnIdToField,
+ observationVariableMappingList,
} from "@langfuse/shared";
import { createW3CTraceId } from "../../utils";
import { isInternalEvalEnvironment } from "../isEvalTargetEnvironmentAllowed";
@@ -28,6 +30,13 @@ interface ScheduleObservationEvalsParams {
configs: ObservationEvalRule[];
schedulerDeps: ObservationEvalSchedulerDeps;
executionMode?: EvalExecutionMode;
+ /**
+ * Extra identity for this scheduling pass so overlapping runs of the same
+ * evaluator on the same observation do not share a job id. Live ingestion
+ * omits it; batch passes the batch-action id so retries of that run stay
+ * stable while a second run gets its own executions.
+ */
+ executionScopeId?: string;
}
/**
@@ -84,7 +93,13 @@ export function isObservationAllowedForQueuedObservationEvals(
export async function scheduleObservationEvals(
params: ScheduleObservationEvalsParams,
): Promise<void> {
- const { observation, configs, schedulerDeps, executionMode } = params;
+ const {
+ observation,
+ configs,
+ schedulerDeps,
+ executionMode,
+ executionScopeId,
+ } = params;
// Early return if no configs
if (configs.length === 0) {
@@ -167,6 +182,7 @@ export async function scheduleObservationEvals(
observationS3Path,
schedulerDeps,
executionMode,
+ executionScopeId,
}).catch((error) => {
logger.error("Failed to process observation eval assignment", {
configId: config.id,
@@ -188,6 +204,7 @@ interface ProcessConfigParams {
observationS3Path: string;
schedulerDeps: ObservationEvalSchedulerDeps;
executionMode?: EvalExecutionMode;
+ executionScopeId?: string;
}
async function processMatchingConfig(
@@ -200,26 +217,28 @@ async function processMatchingConfig(
observationS3Path,
schedulerDeps,
executionMode,
+ executionScopeId,
} = params;
- const jobExecutionId = createW3CTraceId(
- JSON.stringify(
- "assignments" in matchingConfig
- ? [
- "observation-eval",
- matchingConfig.id,
- assignment.id,
- observation.trace_id,
- observation.span_id,
- ]
- : [
- "observation-eval",
- matchingConfig.id,
- observation.trace_id,
- observation.span_id,
- ],
- ),
- );
+ const jobIdentity: string[] =
+ "assignments" in matchingConfig
+ ? [
+ "observation-eval",
+ matchingConfig.id,
+ assignment.id,
+ observation.trace_id,
+ observation.span_id,
+ ]
+ : [
+ "observation-eval",
+ matchingConfig.id,
+ observation.trace_id,
+ observation.span_id,
+ ];
+ if (executionScopeId) {
+ jobIdentity.push(executionScopeId);
+ }
+ const jobExecutionId = createW3CTraceId(JSON.stringify(jobIdentity));
// Create job execution
await schedulerDeps.upsertJobExecution({
@@ -252,6 +271,9 @@ async function processMatchingConfig(
: {}),
}
: {}),
+ ...(assignment.variableMapping != null
+ ? { variableMapping: assignment.variableMapping }
+ : {}),
});
logger.debug("Scheduled observation eval job", {
@@ -269,6 +291,12 @@ type ScheduledObservationEvalAssignment = {
/** Legacy template id, or evaluator id for a ruleless V2 batch run. */
evalTemplateId: string | null;
evaluatorType: ObservationEvalAssignment["evaluator"]["type"];
+ /**
+ * Mapping override for a ruleless batch run. Omitted when the run should
+ * inherit the evaluator version mapping, and never set for rule-backed jobs
+ * (those load the assignment row at pickup).
+ */
+ variableMapping?: ObservationVariableMapping[];
};
function getExecutableAssignments(
@@ -300,6 +328,10 @@ function getExecutableAssignments(
return [];
}
+ const parsedMapping = observationVariableMappingList.safeParse(
+ assignment.variableMapping,
+ );
+
return [
{
id: assignment.id,
@@ -308,6 +340,9 @@ function getExecutableAssignments(
// Ruleless batch runs use the evaluator as the legacy template anchor.
evalTemplateId: rule.ruleId === null ? assignment.evaluator.id : null,
evaluatorType: assignment.evaluator.type,
+ ...(rule.ruleId === null && parsedMapping.success
+ ? { variableMapping: parsedMapping.data }
+ : {}),
},
];
});
diff --git a/worker/src/features/evaluation/observationEval/types.ts b/worker/src/features/evaluation/observationEval/types.ts
index 06bcf4b35e..e156251c1e 100644
--- a/worker/src/features/evaluation/observationEval/types.ts
+++ b/worker/src/features/evaluation/observationEval/types.ts
@@ -8,7 +8,11 @@ import {
JobExecutionStatus,
} from "@langfuse/shared/src/db";
import { type EvalExecutionMode } from "@langfuse/shared";
-import type { EvalTargetObject, FilterState } from "@langfuse/shared";
+import type {
+ EvalTargetObject,
+ FilterState,
+ ObservationVariableMapping,
+} from "@langfuse/shared";
/**
* Re-export ObservationForEval as the canonical observation type for eval operations.
@@ -114,5 +118,10 @@ export interface ObservationEvalSchedulerDeps {
/** Evaluator v2 identity; omitted when scheduling a legacy config. */
evaluatorId?: string;
evaluationRuleId?: string;
+ /**
+ * Mapping override for a ruleless batch run. Omitted to inherit the
+ * evaluator version mapping.
+ */
+ variableMapping?: ObservationVariableMapping[];
}) => Promise<void>;
}