🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(evals): allow overriding variable mapping in batch evaluations (#16975)
Tobias Wochingerc02addc81928
diff --git a/packages/shared/src/features/evals/types.ts b/packages/shared/src/features/evals/types.tsindex e43a4df01c..6edc40ec5b 100644--- a/packages/shared/src/features/evals/types.ts+++ b/packages/shared/src/features/evals/types.ts@@ -317,3 +317,16 @@ export const observationVariableMappingList = z.array(export type ObservationVariableMapping = z.infer<typeof observationVariableMapping>;++/**+ * Per-evaluator mapping override for a one-shot batch evaluation. `null`+ * inherits the evaluator version's mapping. Absent from the payload means+ * every selected evaluator inherits.+ */+export const BatchEvalEvaluatorMappingSchema = z.object({+ evaluatorId: z.string().min(1),+ variableMapping: observationVariableMappingList.nullable(),+});+export type BatchEvalEvaluatorMapping = z.infer<+ typeof BatchEvalEvaluatorMappingSchema+>;diff --git a/packages/shared/src/server/queues.ts b/packages/shared/src/server/queues.tsindex 845f4e0b1e..6c87ba44f2 100644--- a/packages/shared/src/server/queues.ts+++ b/packages/shared/src/server/queues.ts@@ -9,7 +9,11 @@ import { BatchTableNames } from "../interfaces/tableNames";import { EventActionSchema } from "../domain";import { PromptDomainSchema } from "../domain/prompts";import { ObservationAddToDatasetConfigSchema } from "../features/batchAction/addToDatasetTypes";-import { EvalTargetObjectSchema } from "../features/evals/types";+import {+ BatchEvalEvaluatorMappingSchema,+ EvalTargetObjectSchema,+ observationVariableMappingList,+} from "../features/evals/types";import { EvalExecutionMode } from "../features/evals/evalConfigBlocking";import {type MonitorQueueEvent,@@ -157,6 +161,10 @@ export const ObservationEvalExecutionEventSchema = z.object({// which the executor resolves on pickup and records on the execution.evaluatorId: z.string().optional(),evaluationRuleId: z.string().optional(),+ // Ruleless manual batch runs have no assignment row to hold a mapping+ // override. Optional for jobs queued before this field existed; those+ // inherit the evaluator version mapping.+ variableMapping: observationVariableMappingList.optional(),});export const PostHogIntegrationProcessingEventSchema = z.object({projectId: z.string(),@@ -260,6 +268,7 @@ export const BatchActionProcessingEventSchema = z.discriminatedUnion(batchActionId: z.string(),evaluatorIds: z.array(z.string()),evalVersion: z.literal("v2").optional(),+ evaluatorMappings: z.array(BatchEvalEvaluatorMappingSchema).optional(),}),],);diff --git a/web/src/__tests__/server/unit/batch-actions-comment-filter.servertest.ts b/web/src/__tests__/server/unit/batch-actions-comment-filter.servertest.tsindex 3f55c9be01..447a2390f7 100644--- a/web/src/__tests__/server/unit/batch-actions-comment-filter.servertest.ts+++ b/web/src/__tests__/server/unit/batch-actions-comment-filter.servertest.ts@@ -94,6 +94,7 @@ const session = {function prepare({v4BetaEnabled = false,foundEvaluatorIds = [evaluatorId],+ missingPromptVariable = false,} = {}) {const batchActionCreate = vi.fn()@@ -107,7 +108,37 @@ function prepare({findMany: vi.fn(async () => [{ id: evaluatorId }]),},evaluator: {- findMany: vi.fn(async () => foundEvaluatorIds.map((id) => ({ id }))),+ findMany: vi.fn(async (args?: { select?: { id?: boolean } }) => {+ if (args?.select?.id) {+ return foundEvaluatorIds.map((id) => ({ id }));+ }+ return foundEvaluatorIds.map((id) => ({+ id,+ name: "Quality",+ type: "LLM_AS_JUDGE",+ versions: [+ {+ prompt: missingPromptVariable+ ? "Evaluate {{output}} {{input}}"+ : "Evaluate {{output}}",+ promptMessages: [+ {+ role: "user",+ content: missingPromptVariable+ ? "Evaluate {{output}} {{input}}"+ : "Evaluate {{output}}",+ },+ ],+ variableMapping: [+ {+ templateVariable: "output",+ selectedColumnId: "output",+ },+ ],+ },+ ],+ }));+ }),},} as unknown as PrismaClient;const ctx = {@@ -324,4 +355,143 @@ describe("batched evaluation version selection", () => {expect(context.batchActionCreate).not.toHaveBeenCalled();expect(mocks.queueAdd).not.toHaveBeenCalled();});++ it("queues mapping overrides on the batch-eval payload", async () => {+ const context = prepare({ v4BetaEnabled: true });+ const evaluatorMappings = [+ {+ evaluatorId,+ variableMapping: [+ { templateVariable: "output", selectedColumnId: "input" },+ ],+ },+ ];++ await context.runEvaluation.create({+ projectId,+ query,+ evaluatorIds: [evaluatorId],+ sourceTable: BatchEvalSourceTable.EVENTS,+ evalVersion: "v2",+ evaluatorMappings,+ });++ expect(context.prisma.evaluator.findMany).toHaveBeenCalledWith(+ expect.objectContaining({ select: { id: true } }),+ );+ expect(context.prisma.evaluator.findMany).toHaveBeenCalledWith(+ expect.objectContaining({+ include: expect.objectContaining({ versions: expect.anything() }),+ }),+ );+ expect(context.batchActionCreate).toHaveBeenCalledWith(+ expect.objectContaining({+ data: expect.objectContaining({+ config: expect.objectContaining({+ evalVersion: "v2",+ evaluatorMappings,+ }),+ }),+ }),+ );+ expect(mocks.queueAdd).toHaveBeenCalledWith(+ expect.anything(),+ expect.objectContaining({+ payload: expect.objectContaining({+ evalVersion: "v2",+ evaluatorMappings,+ }),+ }),+ expect.anything(),+ );+ });++ it("rejects mapping overrides without evaluator v2", async () => {+ const context = prepare({ v4BetaEnabled: true });++ await expect(+ context.runEvaluation.create({+ projectId,+ query,+ evaluatorIds: [evaluatorId],+ sourceTable: BatchEvalSourceTable.EVENTS,+ evaluatorMappings: [+ {+ evaluatorId,+ variableMapping: [+ { templateVariable: "output", selectedColumnId: "output" },+ ],+ },+ ],+ }),+ ).rejects.toMatchObject({ code: "BAD_REQUEST" });++ expect(context.batchActionCreate).not.toHaveBeenCalled();+ expect(mocks.queueAdd).not.toHaveBeenCalled();+ });++ it("rejects a mapping override for an unselected evaluator", async () => {+ const context = prepare({ v4BetaEnabled: true });++ await expect(+ context.runEvaluation.create({+ projectId,+ query,+ evaluatorIds: [evaluatorId],+ sourceTable: BatchEvalSourceTable.EVENTS,+ evalVersion: "v2",+ evaluatorMappings: [+ {+ evaluatorId: "other-evaluator",+ variableMapping: null,+ },+ ],+ }),+ ).rejects.toMatchObject({ code: "BAD_REQUEST" });++ expect(context.batchActionCreate).not.toHaveBeenCalled();+ expect(mocks.queueAdd).not.toHaveBeenCalled();+ });++ it("rejects more than 100 evaluators", async () => {+ const context = prepare({ v4BetaEnabled: true });++ await expect(+ context.runEvaluation.create({+ projectId,+ query,+ evaluatorIds: Array.from(+ { length: 101 },+ (_, index) => `evaluator-${index}`,+ ),+ sourceTable: BatchEvalSourceTable.EVENTS,+ evalVersion: "v2",+ }),+ ).rejects.toMatchObject({ code: "BAD_REQUEST" });++ expect(context.batchActionCreate).not.toHaveBeenCalled();+ });++ it("names the evaluator when a mapping is incomplete", async () => {+ const context = prepare({+ v4BetaEnabled: true,+ missingPromptVariable: true,+ });++ await expect(+ context.runEvaluation.create({+ projectId,+ query,+ evaluatorIds: [evaluatorId],+ sourceTable: BatchEvalSourceTable.EVENTS,+ evalVersion: "v2",+ evaluatorMappings: [{ evaluatorId, variableMapping: null }],+ }),+ ).rejects.toMatchObject({+ code: "BAD_REQUEST",+ message: expect.stringContaining('Evaluator "Quality"'),+ });++ expect(context.batchActionCreate).not.toHaveBeenCalled();+ });});diff --git a/web/src/features/batch-actions/components/RunEvaluationDialog/ConfirmationStep.tsx b/web/src/features/batch-actions/components/RunEvaluationDialog/ConfirmationStep.tsxindex f5fb951a1f..4ba16dbb8d 100644--- a/web/src/features/batch-actions/components/RunEvaluationDialog/ConfirmationStep.tsx+++ b/web/src/features/batch-actions/components/RunEvaluationDialog/ConfirmationStep.tsx@@ -2,7 +2,8 @@ import { Card, CardContent } from "@/src/components/ui/card";import { Badge } from "@/src/components/ui/badge";import { Separator } from "@/src/components/ui/separator";import { EstimatedCostRow } from "./EstimatedCostRow";-import { BatchEvalSourceTable } from "@langfuse/shared";+import type { BatchEvalSourceTable } from "@langfuse/shared";+import { getBatchEvalCostObservationCount } from "./utils";type ConfirmationStepProps = {projectId: string;@@ -10,53 +11,15 @@ type ConfirmationStepProps = {evaluators: Array<{ id: string; name: string }>;hideCount: boolean;sourceTable: BatchEvalSourceTable;- experimentCount?: number;};-/**- * Determines whether to show a cost disclaimer instead of the actual cost estimate.- * For experiments source, we can't accurately estimate cost because displayCount- * is the experiment count, not the actual observation count.- */-function shouldShowCostDisclaimer(sourceTable: BatchEvalSourceTable): boolean {- return sourceTable === BatchEvalSourceTable.EXPERIMENTS;-}--/**- * Calculates the effective observation count for cost estimation.- * For experiment-items source, multiplies by experiment count since each item- * is evaluated once per experiment.- */-function getEffectiveObservationCount(- displayCount: number,- sourceTable: BatchEvalSourceTable,- experimentCount?: number,-): number {- if (- sourceTable === BatchEvalSourceTable.EXPERIMENT_ITEMS &&- experimentCount- ) {- return displayCount * experimentCount;- }- return displayCount;-}-export function ConfirmationStep(props: ConfirmationStepProps) {- const {- projectId,- displayCount,- evaluators,- hideCount,- sourceTable,- experimentCount,- } = props;+ const { projectId, displayCount, evaluators, hideCount, sourceTable } = props;- const showCostDisclaimer = shouldShowCostDisclaimer(sourceTable);- const effectiveObservationCount = getEffectiveObservationCount(+ const effectiveObservationCount = getBatchEvalCostObservationCount({displayCount,sourceTable,- experimentCount,- );+ });return (<div className="space-y-4">@@ -86,7 +49,7 @@ export function ConfirmationStep(props: ConfirmationStepProps) {<Separator />- {showCostDisclaimer ? (+ {effectiveObservationCount == null ? (<div className="flex gap-2"><span className="text-muted-foreground shrink-0">Est. LLM API Key Cost:diff --git a/web/src/features/batch-actions/components/RunEvaluationDialog/EvaluatorMappingStep.tsx b/web/src/features/batch-actions/components/RunEvaluationDialog/EvaluatorMappingStep.tsxnew file mode 100644index 0000000000..863c868f45--- /dev/null+++ b/web/src/features/batch-actions/components/RunEvaluationDialog/EvaluatorMappingStep.tsx@@ -0,0 +1,226 @@+import { useStore } from "zustand";+import type { ReactNode } from "react";++import { Card, CardContent } from "@/src/components/ui/card";+import { InfoTooltip } from "@/src/components/ui/InfoTooltip/InfoTooltip";+import { Skeleton } from "@/src/components/ui/skeleton";+import {+ EvaluatorAssignmentsEditor,+ type RuleEvaluatorOption,+} from "@/src/features/evals";+import {+ useRuleCostEstimate,+ type RuleCostEstimate,+} from "@/src/features/evals/v2/hooks/useRuleCostEstimate";+import type { RuleSetupStore } from "@/src/features/evals/v2/types/rules";+import { usdFormatter } from "@/src/utils/numbers";++export function EvaluatorMappingStep({+ projectId,+ store,+ evaluatorOptions,+ isQueryLoading,+ isQueryError,+ queryErrorMessage,+ search,+ onSearchChange,+ sampleObject,+ costObservationCount,+}: {+ projectId: string;+ store: RuleSetupStore;+ evaluatorOptions: RuleEvaluatorOption[];+ isQueryLoading: boolean;+ isQueryError: boolean;+ queryErrorMessage: string | undefined;+ search: string;+ onSearchChange: (value: string) => void;+ sampleObject: Record<string, unknown> | null;+ costObservationCount: number | null;+}) {+ if (costObservationCount == null) {+ return (+ <EvaluatorMappingEditor+ store={store}+ evaluatorOptions={evaluatorOptions}+ isQueryLoading={isQueryLoading}+ isQueryError={isQueryError}+ queryErrorMessage={queryErrorMessage}+ search={search}+ onSearchChange={onSearchChange}+ sampleObject={sampleObject}+ costEstimates={[]}+ estimatingEvaluatorIds={[]}+ footerTrailing={+ <p className="text-muted-foreground max-w-64 text-right text-xs">+ Cost estimate unavailable for experiment-scoped evaluations+ </p>+ }+ costError={false}+ />+ );+ }++ return (+ <EvaluatorMappingStepWithCosts+ projectId={projectId}+ store={store}+ evaluatorOptions={evaluatorOptions}+ isQueryLoading={isQueryLoading}+ isQueryError={isQueryError}+ queryErrorMessage={queryErrorMessage}+ search={search}+ onSearchChange={onSearchChange}+ sampleObject={sampleObject}+ costObservationCount={costObservationCount}+ />+ );+}++function EvaluatorMappingStepWithCosts({+ projectId,+ store,+ evaluatorOptions,+ isQueryLoading,+ isQueryError,+ queryErrorMessage,+ search,+ onSearchChange,+ sampleObject,+ costObservationCount,+}: {+ projectId: string;+ store: RuleSetupStore;+ evaluatorOptions: RuleEvaluatorOption[];+ isQueryLoading: boolean;+ isQueryError: boolean;+ queryErrorMessage: string | undefined;+ search: string;+ onSearchChange: (value: string) => void;+ sampleObject: Record<string, unknown> | null;+ costObservationCount: number;+}) {+ const assignments = useStore(store, (state) => state.assignments);+ const costEstimate = useRuleCostEstimate({+ projectId,+ store,+ matchingObservations: costObservationCount,+ });+ const availableEstimates = costEstimate.estimates.filter(+ (estimate) => estimate.estimatedCostUsd !== null,+ );+ const totalCostUsd = availableEstimates.reduce(+ (total, estimate) => total + (estimate.estimatedCostUsd ?? 0),+ 0,+ );+ const footerTrailing =+ assignments.length > 0 ? (+ <div className="w-48 border-t pt-2 text-right text-sm">+ {availableEstimates.length > 0 ? (+ <div>+ <div className="flex items-center justify-end gap-1.5">+ <p className="font-mono leading-none font-bold whitespace-nowrap tabular-nums">+ ≈ {usdFormatter(totalCostUsd, 2, 2)}+ </p>+ <InfoTooltip label="About total estimated LLM costs">+ Sum of the available LLM cost estimates for this run.+ </InfoTooltip>+ </div>+ <p className="text-muted-foreground text-xs whitespace-nowrap">+ estimated LLM costs for this run+ </p>+ </div>+ ) : costEstimate.status === "estimating" ? (+ <Skeleton className="ml-auto h-5 w-28" />+ ) : (+ <div>+ <p className="font-mono font-bold">Unavailable</p>+ <p className="text-muted-foreground text-xs whitespace-nowrap">+ estimated LLM costs for this run+ </p>+ </div>+ )}+ </div>+ ) : null;++ return (+ <EvaluatorMappingEditor+ store={store}+ evaluatorOptions={evaluatorOptions}+ isQueryLoading={isQueryLoading}+ isQueryError={isQueryError}+ queryErrorMessage={queryErrorMessage}+ search={search}+ onSearchChange={onSearchChange}+ sampleObject={sampleObject}+ costEstimates={costEstimate.estimates}+ estimatingEvaluatorIds={costEstimate.estimatingEvaluatorIds}+ footerTrailing={footerTrailing}+ costError={costEstimate.status === "error" && assignments.length > 0}+ />+ );+}++function EvaluatorMappingEditor({+ store,+ evaluatorOptions,+ isQueryLoading,+ isQueryError,+ queryErrorMessage,+ search,+ onSearchChange,+ sampleObject,+ costEstimates,+ estimatingEvaluatorIds,+ footerTrailing,+ costError,+}: {+ store: RuleSetupStore;+ evaluatorOptions: RuleEvaluatorOption[];+ isQueryLoading: boolean;+ isQueryError: boolean;+ queryErrorMessage: string | undefined;+ search: string;+ onSearchChange: (value: string) => void;+ sampleObject: Record<string, unknown> | null;+ costEstimates: RuleCostEstimate[];+ estimatingEvaluatorIds: string[];+ footerTrailing: ReactNode;+ costError: boolean;+}) {+ return (+ <div className="flex h-full min-h-0 flex-col gap-3">+ <div className="min-h-0 flex-1 overflow-y-auto">+ {isQueryLoading ? (+ <p className="text-muted-foreground text-sm">Loading evaluators...</p>+ ) : isQueryError ? (+ <Card>+ <CardContent className="text-destructive p-4 text-sm">+ Failed to load evaluators: {queryErrorMessage}+ </CardContent>+ </Card>+ ) : (+ <>+ <EvaluatorAssignmentsEditor+ evaluatorOptions={evaluatorOptions}+ store={store}+ search={search}+ onSearchChange={onSearchChange}+ sampleObject={sampleObject}+ costEstimates={costEstimates}+ estimatingEvaluatorIds={estimatingEvaluatorIds}+ footerTrailing={footerTrailing}+ emptyDescription="Attach an evaluator to score this selection."+ sourceUnavailableMessage="No observation is available to validate JSON paths."+ />+ {costError ? (+ <p className="text-muted-foreground mt-2 text-xs">+ Cost estimates are temporarily unavailable.+ </p>+ ) : null}+ </>+ )}+ </div>+ </div>+ );+}diff --git a/web/src/features/batch-actions/components/RunEvaluationDialog/EvaluatorSelectionStep.tsx b/web/src/features/batch-actions/components/RunEvaluationDialog/EvaluatorSelectionStep.tsxindex dd8b29dd03..b11bf31f94 100644--- a/web/src/features/batch-actions/components/RunEvaluationDialog/EvaluatorSelectionStep.tsx+++ b/web/src/features/batch-actions/components/RunEvaluationDialog/EvaluatorSelectionStep.tsx@@ -1,5 +1,4 @@import { useMemo } from "react";-import Link from "next/link";import {observationVariableMappingList,type ObservationVariableMapping,@@ -12,7 +11,7 @@ import { Checkbox } from "@/src/components/design-system/Checkbox/Checkbox";import { Input } from "@/src/components/ui/input";import { EvaluatorPromptPreview } from "./EvaluatorPromptPreview";import { renderPromptPreviewFromObservation } from "./utils";-import { ExternalLink, Eye, Plus, X } from "lucide-react";+import { Eye, X } from "lucide-react";export type BatchEvaluator = {id: string;@@ -34,7 +33,6 @@ type EvaluatorSelectionStepProps = {evaluatorSearchQuery: string;onSearchQueryChange: (query: string) => void;onToggleEvaluator: (evaluatorId: string) => void;- createEvaluatorHref: string;};export function EvaluatorSelectionStep(props: EvaluatorSelectionStepProps) {@@ -50,7 +48,6 @@ export function EvaluatorSelectionStep(props: EvaluatorSelectionStepProps) {evaluatorSearchQuery,onSearchQueryChange,onToggleEvaluator,- createEvaluatorHref,} = props;const filteredEvaluators = useMemo(() => {@@ -228,19 +225,6 @@ export function EvaluatorSelectionStep(props: EvaluatorSelectionStepProps) {</div>)}</div>-- <Button variant="outline" size="default" className="h-9 w-full" asChild>- <Link- href={createEvaluatorHref}- target="_blank"- rel="noreferrer"- aria-label="Create new Evaluator (opens in a new tab)"- >- <Plus className="mr-1 h-4 w-4" />- Create new Evaluator- <ExternalLink className="ml-1 h-3.5 w-3.5" aria-hidden="true" />- </Link>- </Button></div>);}diff --git a/web/src/features/batch-actions/components/RunEvaluationDialog/index.tsx b/web/src/features/batch-actions/components/RunEvaluationDialog/index.tsxindex 6f0ef133c3..683ce4fd2f 100644--- a/web/src/features/batch-actions/components/RunEvaluationDialog/index.tsx+++ b/web/src/features/batch-actions/components/RunEvaluationDialog/index.tsx@@ -1,8 +1,11 @@import { useMemo, useState } from "react";+import Link from "next/link";+import { useStore } from "zustand";import {type BatchActionQuery,type BatchEvalSourceTable,BatchEvalSourceTable as SourceTable,+ extractVariables,observationVariableMappingList,} from "@langfuse/shared";import { api, sendAsPostOption } from "@/src/utils/api";@@ -16,15 +19,35 @@ import {DialogTitle,} from "@/src/components/ui/dialog";import { Button } from "@/src/components/ui/button";+import { Skeleton } from "@/src/components/ui/skeleton";+import {+ Tooltip,+ TooltipContent,+ TooltipTrigger,+} from "@/src/components/ui/tooltip";import { showErrorToast, showSuccessToast } from "@/src/features/notifications";-import { ChevronLeft } from "lucide-react";+import { ChevronLeft, ExternalLink, Plus } from "lucide-react";import {EvaluatorSelectionStep,type BatchEvaluator,} from "./EvaluatorSelectionStep";+import { EvaluatorMappingStep } from "./EvaluatorMappingStep";import { ConfirmationStep } from "./ConfirmationStep";-import { buildQueryWithSelectedIds, getCreateEvaluatorHref } from "./utils";-import { useForceV3Experience } from "@/src/features/v4-migration/useForceV3Experience";+import {+ buildQueryWithSelectedIds,+ getBatchEvalCostObservationCount,+ getCreateEvaluatorHref,+ hasCompleteBatchEvalMappings,+} from "./utils";+import { BATCH_EVAL_EVALUATOR_LIMIT } from "@/src/features/batch-actions/validation";+import {+ buildSelectedSampleObject,+ createRuleSetupStore,+} from "@/src/features/evals";+import { coverEvaluatorPromptVariables } from "@/src/features/evals/v2/fns/variableMapping/coverEvaluatorPromptVariables";+import { prepareModernRuleVariableMapping } from "@/src/features/evals/v2/fns/variableMapping/prepareModernRuleVariableMapping";+import { useDebounce } from "@/src/hooks/useDebounce";+import { usePostHogClientCapture } from "@/src/features/posthog-analytics";type RunEvaluationDialogProps = {projectId: string;@@ -44,9 +67,6 @@ type RunEvaluationDialogProps = {type DialogStep = "select-evaluator" | "confirm";-/** Matches the evaluator overview page size; the step filters client-side. */-const BATCH_EVALUATOR_LIMIT = 100;-export function RunEvaluationDialog(props: RunEvaluationDialogProps) {const {projectId,@@ -57,20 +77,53 @@ export function RunEvaluationDialog(props: RunEvaluationDialogProps) {sourceTable = SourceTable.EVENTS,} = props;+ const capture = usePostHogClientCapture();const [step, setStep] = useState<DialogStep>("select-evaluator");const [selectedEvaluators, setSelectedEvaluators] = useState<BatchEvaluator[]>([]);const [evaluatorSearchQuery, setEvaluatorSearchQuery] = useState("");- const forceV3Experience = useForceV3Experience(projectId);+ const [mappingSearch, setMappingSearch] = useState("");+ const [mappingSearchQuery, setMappingSearchQuery] = useState("");+ const debouncedMappingSearch = useDebounce(setMappingSearchQuery, 300, false);+ const [ruleSetupStore] = useState(() =>+ createRuleSetupStore({+ name: "Batch evaluation",+ filter: [],+ sampling: 1,+ assignments: [],+ }),+ );+ const mappingAssignments = useStore(+ ruleSetupStore,+ (state) => state.assignments,+ );- // Unsearched: `EvaluatorSelectionStep` filters the list client-side, the- // same way the overview does, so typing does not refetch.- const evaluatorsQuery = api.evalsV2.options.useQuery({- projectId,- limit: BATCH_EVALUATOR_LIMIT,- excludeLegacyEvaluators: true,- });+ const forceV3Query = api.v4Transition.forceV3Experience.useQuery(+ { projectId },+ {+ enabled: Boolean(projectId),+ staleTime: Infinity,+ },+ );+ const forceV3Experience = forceV3Query.data === true;+ const isExperiencePending = forceV3Query.isPending;+ const showMappingEditor = !isExperiencePending && !forceV3Experience;++ const evaluatorsQuery = api.evalsV2.options.useQuery(+ {+ projectId,+ limit: BATCH_EVAL_EVALUATOR_LIMIT,+ excludeLegacyEvaluators: true,+ search:+ showMappingEditor && mappingSearchQuery.trim()+ ? mappingSearchQuery.trim()+ : undefined,+ },+ {+ placeholderData: (previousData) => previousData,+ },+ );const runEvaluationMutation =api.batchAction.runEvaluation.create.useMutation({@@ -80,7 +133,6 @@ export function RunEvaluationDialog(props: RunEvaluationDialogProps) {});const displayCount = selectAll ? totalCount : selectedObservationIds.length;- // For experiments source, displayCount is experiment count, not item countconst isExperimentsSource = sourceTable === SourceTable.EXPERIMENTS;const scopeLabel =sourceTable === SourceTable.EVENTS ? "observation" : "experiment item";@@ -115,8 +167,6 @@ export function RunEvaluationDialog(props: RunEvaluationDialogProps) {const eligibleEvaluators = useMemo(() =>- // A blocked evaluator is skipped by the scheduler, so offering it would- // report a successful batch that produced no scores.(evaluatorsQuery.data ?? []).filter((evaluator) => evaluator.blockedAt === null).map(@@ -133,11 +183,62 @@ export function RunEvaluationDialog(props: RunEvaluationDialogProps) {[evaluatorsQuery.data],);+ const evaluatorOptions = useMemo(+ () =>+ (evaluatorsQuery.data ?? [])+ .filter((evaluator) => evaluator.blockedAt === null)+ .map((evaluator) => {+ const prepared = prepareModernRuleVariableMapping(+ evaluator.latestVersion?.variableMapping,+ evaluator.type,+ );+ const requiredVariables =+ evaluator.type === "CODE"+ ? []+ : extractVariables(evaluator.latestVersion?.prompt ?? "");++ return {+ id: evaluator.id,+ name: evaluator.name,+ type: evaluator.type,+ updatedAt: evaluator.updatedAt,+ createdByUser: evaluator.createdByUser,+ defaultVariableMapping: coverEvaluatorPromptVariables(+ prepared.defaultVariableMapping,+ requiredVariables,+ ),+ initialVariableMapping: prepared.initialVariableMapping,+ requiredVariables,+ };+ }),+ [evaluatorsQuery.data],+ );+const selectedEvaluatorIds = useMemo(() => selectedEvaluators.map((evaluator) => evaluator.id),[selectedEvaluators],);+ const sampleObject = buildSelectedSampleObject({+ observation: props.exampleObservation?.id ? props.exampleObservation : null,+ eventDetails: previewQuery.data?.[0],+ });++ const selectedCount = showMappingEditor+ ? mappingAssignments.length+ : selectedEvaluators.length;+ const mappingsComplete = hasCompleteBatchEvalMappings(mappingAssignments);+ const costObservationCount = getBatchEvalCostObservationCount({+ displayCount,+ sourceTable,+ });+ const mappingRunDisabledReason =+ selectedCount === 0+ ? "Attach at least one evaluator."+ : mappingsComplete+ ? null+ : "Map every evaluator variable to a source column before running.";+const toggleEvaluatorSelection = (evaluatorId: string) => {setSelectedEvaluators((previous) => {if (previous.some((evaluator) => evaluator.id === evaluatorId)) {@@ -151,7 +252,10 @@ export function RunEvaluationDialog(props: RunEvaluationDialogProps) {};const onSubmit = async () => {- if (selectedEvaluators.length === 0) {+ if (selectedCount === 0) {+ return;+ }+ if (showMappingEditor && !mappingsComplete) {return;}@@ -160,26 +264,47 @@ export function RunEvaluationDialog(props: RunEvaluationDialogProps) {selectAll,selectedObservationIds,});+ const evaluatorIds = showMappingEditor+ ? mappingAssignments.map((assignment) => assignment.evaluatorId)+ : selectedEvaluators.map((evaluator) => evaluator.id);+ const evaluatorMappings = showMappingEditor+ ? mappingAssignments.map((assignment) => ({+ evaluatorId: assignment.evaluatorId,+ variableMapping: assignment.variableMapping,+ }))+ : undefined;+ const mappingOverrideCount = (evaluatorMappings ?? []).filter(+ (mapping) => mapping.variableMapping !== null,+ ).length;try {await runEvaluationMutation.mutateAsync({projectId,query: finalQuery,- evaluatorIds: selectedEvaluators.map((evaluator) => evaluator.id),+ evaluatorIds,sourceTable,evalVersion: "v2",+ ...(evaluatorMappings ? { evaluatorMappings } : {}),});} catch {return;}+ capture("batch_eval:run", {+ evaluatorCount: evaluatorIds.length,+ mappingOverrideCount,+ sourceTable,+ isForceV3Experience: forceV3Experience,+ isV4: true,+ });+showSuccessToast({title: "Evaluation queued",description: isExperimentsSource- ? `Scheduled evaluation for items from ${displayCount} selected experiment${displayCount === 1 ? "" : "s"} with ${selectedEvaluators.length} ${selectedEvaluators.length === 1 ? "evaluator" : "evaluators"}.`+ ? `Scheduled evaluation for items from ${displayCount} selected experiment${displayCount === 1 ? "" : "s"} with ${evaluatorIds.length} ${evaluatorIds.length === 1 ? "evaluator" : "evaluators"}.`: sourceTable === SourceTable.EXPERIMENT_ITEMS- ? `Scheduled evaluation for up to ${displayCount} experiment item${displayCount === 1 ? "" : "s"} across ${experimentItemsExperimentCount} experiment${experimentItemsExperimentCount === 1 ? "" : "s"} with ${selectedEvaluators.length} ${selectedEvaluators.length === 1 ? "evaluator" : "evaluators"}.`- : `Scheduled evaluation for ${displayCount} selected ${scopeLabel}${displayCount === 1 ? "" : "s"} with ${selectedEvaluators.length} ${selectedEvaluators.length === 1 ? "evaluator" : "evaluators"}.`,+ ? `Scheduled evaluation for up to ${displayCount} experiment item${displayCount === 1 ? "" : "s"} across ${experimentItemsExperimentCount} experiment${experimentItemsExperimentCount === 1 ? "" : "s"} with ${evaluatorIds.length} ${evaluatorIds.length === 1 ? "evaluator" : "evaluators"}.`+ : `Scheduled evaluation for ${displayCount} selected ${scopeLabel}${displayCount === 1 ? "" : "s"} with ${evaluatorIds.length} ${evaluatorIds.length === 1 ? "evaluator" : "evaluators"}.`,link: {href: `/project/${projectId}/settings/batch-actions`,text: "View batch actions",@@ -189,10 +314,22 @@ export function RunEvaluationDialog(props: RunEvaluationDialogProps) {props.onClose();};+ const createEvaluatorHref = getCreateEvaluatorHref({+ projectId,+ forceV3Experience,+ });+return (<><Dialog open onOpenChange={(open) => !open && props.onClose()}>- <DialogContent className="flex max-h-[62vh] min-h-[38vh] max-w-2xl flex-col">+ <DialogContent+ {...(showMappingEditor ? { size: "lg" as const } : {})}+ className={+ showMappingEditor+ ? "flex max-h-[85vh] min-h-[38vh] flex-col"+ : "flex max-h-[62vh] min-h-[38vh] max-w-2xl flex-col"+ }+ ><DialogHeader><DialogTitle>{isExperimentsSource@@ -204,40 +341,63 @@ export function RunEvaluationDialog(props: RunEvaluationDialogProps) {<DialogDescription>{step === "confirm"? "Review your evaluation configuration before running."- : "Select one or more evaluators."}+ : showMappingEditor+ ? "Select evaluators and review their variable mappings."+ : "Select one or more evaluators."}</DialogDescription></DialogHeader>- <DialogBody className="flex-1 overflow-hidden">- {step === "select-evaluator" ? (- <EvaluatorSelectionStep- eligibleEvaluators={eligibleEvaluators}- selectedEvaluators={selectedEvaluators}- isQueryLoading={evaluatorsQuery.isLoading}- isQueryError={evaluatorsQuery.isError}- queryErrorMessage={evaluatorsQuery.error?.message}- previewObservation={previewQuery.data?.[0]}- isPreviewLoading={previewQuery.isLoading}- selectedEvaluatorIds={selectedEvaluatorIds}- evaluatorSearchQuery={evaluatorSearchQuery}- onSearchQueryChange={setEvaluatorSearchQuery}- onToggleEvaluator={toggleEvaluatorSelection}- createEvaluatorHref={getCreateEvaluatorHref({- projectId,- forceV3Experience,- })}- />+ <DialogBody+ className={+ showMappingEditor+ ? "min-h-0 flex-1 overflow-y-auto"+ : "flex-1 overflow-hidden"+ }+ >+ {isExperiencePending ? (+ <Skeleton className="h-20 w-full" />+ ) : step === "select-evaluator" ? (+ showMappingEditor ? (+ <EvaluatorMappingStep+ projectId={projectId}+ store={ruleSetupStore}+ evaluatorOptions={evaluatorOptions}+ isQueryLoading={evaluatorsQuery.isLoading}+ isQueryError={evaluatorsQuery.isError}+ queryErrorMessage={evaluatorsQuery.error?.message}+ search={mappingSearch}+ onSearchChange={(value) => {+ setMappingSearch(value);+ debouncedMappingSearch(value);+ }}+ sampleObject={sampleObject}+ costObservationCount={costObservationCount}+ />+ ) : (+ <EvaluatorSelectionStep+ eligibleEvaluators={eligibleEvaluators}+ selectedEvaluators={selectedEvaluators}+ isQueryLoading={evaluatorsQuery.isLoading}+ isQueryError={evaluatorsQuery.isError}+ queryErrorMessage={evaluatorsQuery.error?.message}+ previewObservation={previewQuery.data?.[0]}+ isPreviewLoading={previewQuery.isLoading}+ selectedEvaluatorIds={selectedEvaluatorIds}+ evaluatorSearchQuery={evaluatorSearchQuery}+ onSearchQueryChange={setEvaluatorSearchQuery}+ onToggleEvaluator={toggleEvaluatorSelection}+ />+ )) : (<ConfirmationStepprojectId={projectId}displayCount={displayCount}- evaluators={selectedEvaluators.map((e) => ({- id: e.id,- name: e.scoreName,+ evaluators={selectedEvaluators.map((evaluator) => ({+ id: evaluator.id,+ name: evaluator.scoreName,}))}hideCount={sourceTable !== SourceTable.EVENTS}sourceTable={sourceTable}- experimentCount={experimentItemsExperimentCount}/>)}</DialogBody>@@ -256,27 +416,95 @@ export function RunEvaluationDialog(props: RunEvaluationDialogProps) {<div />)}- {step === "select-evaluator" ? (- <Button- onClick={() => setStep("confirm")}- disabled={selectedEvaluators.length === 0}- >- Continue{" "}- {selectedEvaluators.length > 0- ? `with ${selectedEvaluators.length} evaluator(s)`- : null}- </Button>- ) : (- <Button- onClick={onSubmit}- loading={runEvaluationMutation.isPending}- >- Run Evaluation- </Button>- )}+ <div className="flex items-center gap-2">+ {step !== "confirm" ? (+ <CreateEvaluatorButton href={createEvaluatorHref} />+ ) : null}+ {showMappingEditor ? (+ <MappingRunButton+ disabledReason={mappingRunDisabledReason}+ selectedCount={selectedCount}+ loading={runEvaluationMutation.isPending}+ onClick={onSubmit}+ />+ ) : step === "select-evaluator" ? (+ <Button+ onClick={() => setStep("confirm")}+ disabled={isExperiencePending || selectedCount === 0}+ >+ Continue{" "}+ {selectedCount > 0+ ? `with ${selectedCount} evaluator(s)`+ : null}+ </Button>+ ) : (+ <Button+ onClick={onSubmit}+ loading={runEvaluationMutation.isPending}+ >+ Run Evaluation+ </Button>+ )}+ </div></DialogFooter></DialogContent></Dialog></>);}++function CreateEvaluatorButton({ href }: { href: string }) {+ return (+ <Button variant="secondary" className="gap-1.5" asChild>+ <Link+ href={href}+ target="_blank"+ rel="noreferrer"+ aria-label="Create new Evaluator (opens in a new tab)"+ >+ <Plus className="size-4 shrink-0" aria-hidden="true" />+ Create new Evaluator+ <ExternalLink className="size-3.5 shrink-0" aria-hidden="true" />+ </Link>+ </Button>+ );+}++function MappingRunButton({+ disabledReason,+ selectedCount,+ loading,+ onClick,+}: {+ disabledReason: string | null;+ selectedCount: number;+ loading: boolean;+ onClick: () => void;+}) {+ const button = (+ <Button+ onClick={onClick}+ disabled={Boolean(disabledReason)}+ loading={loading}+ className={disabledReason ? "pointer-events-none" : undefined}+ >+ Run Evaluation+ {selectedCount > 0 ? ` with ${selectedCount} evaluator(s)` : null}+ </Button>+ );++ if (!disabledReason) {+ return button;+ }++ return (+ <Tooltip>+ <TooltipTrigger asChild>+ <span className="inline-flex cursor-not-allowed" tabIndex={0}>+ {button}+ </span>+ </TooltipTrigger>+ <TooltipContent>{disabledReason}</TooltipContent>+ </Tooltip>+ );+}diff --git a/web/src/features/batch-actions/components/RunEvaluationDialog/utils.clienttest.ts b/web/src/features/batch-actions/components/RunEvaluationDialog/utils.clienttest.tsindex e1b4444e35..1dbaea4208 100644--- a/web/src/features/batch-actions/components/RunEvaluationDialog/utils.clienttest.ts+++ b/web/src/features/batch-actions/components/RunEvaluationDialog/utils.clienttest.ts@@ -1,5 +1,10 @@import { describe, expect, it } from "vitest";-import { getCreateEvaluatorHref } from "./utils";+import { BatchEvalSourceTable, EvalTemplateType } from "@langfuse/shared";+import {+ getBatchEvalCostObservationCount,+ getCreateEvaluatorHref,+ hasCompleteBatchEvalMappings,+} from "./utils";describe("getCreateEvaluatorHref", () => {it("links forced-v3 projects to the legacy evaluator UI", () => {@@ -20,3 +25,102 @@ describe("getCreateEvaluatorHref", () => {).toBe("/project/project-id/evals?gallery=open");});});++describe("getBatchEvalCostObservationCount", () => {+ it("returns the selected observation count for events", () => {+ expect(+ getBatchEvalCostObservationCount({+ displayCount: 12,+ sourceTable: BatchEvalSourceTable.EVENTS,+ }),+ ).toBe(12);+ });++ it("uses the already-expanded observation count for experiment items", () => {+ expect(+ getBatchEvalCostObservationCount({+ displayCount: 12,+ sourceTable: BatchEvalSourceTable.EXPERIMENT_ITEMS,+ }),+ ).toBe(12);+ });++ it("returns null when the observation count is unknown", () => {+ expect(+ getBatchEvalCostObservationCount({+ displayCount: 5,+ sourceTable: BatchEvalSourceTable.EXPERIMENTS,+ }),+ ).toBeNull();+ });+});++describe("hasCompleteBatchEvalMappings", () => {+ it("treats code evaluators as complete without a mapping", () => {+ expect(+ hasCompleteBatchEvalMappings([+ {+ evaluatorType: EvalTemplateType.CODE,+ variableMapping: null,+ defaultVariableMapping: [],+ },+ ]),+ ).toBe(true);+ });++ it("rejects an LLM mapping with an empty source column", () => {+ expect(+ hasCompleteBatchEvalMappings([+ {+ evaluatorType: EvalTemplateType.LLM_AS_JUDGE,+ variableMapping: [+ { templateVariable: "output", selectedColumnId: "" },+ ],+ defaultVariableMapping: [+ { templateVariable: "output", selectedColumnId: "output" },+ ],+ },+ ]),+ ).toBe(false);+ });++ it("accepts inherited mappings that already have source columns", () => {+ expect(+ hasCompleteBatchEvalMappings([+ {+ evaluatorType: EvalTemplateType.LLM_AS_JUDGE,+ variableMapping: null,+ defaultVariableMapping: [+ { templateVariable: "output", selectedColumnId: "output" },+ ],+ },+ ]),+ ).toBe(true);+ });++ it("rejects an empty LLM mapping when the prompt still has variables", () => {+ expect(+ hasCompleteBatchEvalMappings([+ {+ evaluatorType: EvalTemplateType.LLM_AS_JUDGE,+ variableMapping: null,+ defaultVariableMapping: [],+ requiredVariables: ["user_input", "assistant_output"],+ },+ ]),+ ).toBe(false);+ });++ it("accepts an LLM evaluator whose prompt has no variables", () => {+ expect(+ hasCompleteBatchEvalMappings([+ {+ evaluatorType: EvalTemplateType.LLM_AS_JUDGE,+ variableMapping: null,+ defaultVariableMapping: [],+ requiredVariables: [],+ },+ ]),+ ).toBe(true);+ });+});diff --git a/web/src/features/batch-actions/components/RunEvaluationDialog/utils.ts b/web/src/features/batch-actions/components/RunEvaluationDialog/utils.tsindex 752cb71829..9dbd2bf9c9 100644--- a/web/src/features/batch-actions/components/RunEvaluationDialog/utils.ts+++ b/web/src/features/batch-actions/components/RunEvaluationDialog/utils.ts@@ -1,4 +1,6 @@import {+ BatchEvalSourceTable,+ EvalTemplateType,extractValueFromObjectAsString,zipToolCallsFromRecord,type BatchActionQuery,@@ -82,6 +84,53 @@ export function renderPromptPreviewFromObservation(params: {: renderedPrompt;}+/**+ * Observations this batch will score, or null when the count cannot be known+ * up front (experiment-scoped runs).+ *+ * `displayCount` is already the observation count for experiment items:+ * callers expand selected rows / select-all totals across experiments.+ */+export function getBatchEvalCostObservationCount(params: {+ displayCount: number;+ sourceTable: BatchEvalSourceTable;+}): number | null {+ const { displayCount, sourceTable } = params;+ if (sourceTable === BatchEvalSourceTable.EXPERIMENTS) {+ return null;+ }+ return displayCount;+}++export function hasCompleteBatchEvalMappings(+ assignments: Array<{+ evaluatorType: EvalTemplateType;+ variableMapping: ObservationVariableMapping[] | null;+ defaultVariableMapping: ObservationVariableMapping[];+ requiredVariables?: string[];+ }>,+): boolean {+ return assignments.every((assignment) => {+ if (assignment.evaluatorType === EvalTemplateType.CODE) {+ return true;+ }+ const mapping =+ assignment.variableMapping ?? assignment.defaultVariableMapping;+ const mappedVariables = new Set(+ mapping+ .filter((entry) => Boolean(entry.selectedColumnId?.trim()))+ .map((entry) => entry.templateVariable),+ );+ const requiredVariables = assignment.requiredVariables ?? [];+ if (requiredVariables.length > 0) {+ return requiredVariables.every((variable) =>+ mappedVariables.has(variable),+ );+ }+ return mapping.every((entry) => Boolean(entry.selectedColumnId?.trim()));+ });+}+export function buildQueryWithSelectedIds(params: {query: BatchActionQuery;selectAll: boolean;diff --git a/web/src/features/batch-actions/server/prepareBatchEvalEvaluatorMappings.ts b/web/src/features/batch-actions/server/prepareBatchEvalEvaluatorMappings.tsnew file mode 100644index 0000000000..624f4245d8--- /dev/null+++ b/web/src/features/batch-actions/server/prepareBatchEvalEvaluatorMappings.ts@@ -0,0 +1,91 @@+import {+ EvalTemplateType,+ InvalidRequestError,+ type BatchEvalEvaluatorMapping,+} from "@langfuse/shared";+import type { PrismaClient } from "@langfuse/shared/src/db";+import { findEvaluatorsByIds } from "@/src/features/evals/v2/server/evaluators/evaluatorRepository";+import { reconcileEvaluatorPromptMessages } from "@/src/features/evals/v2/server/evaluators/evaluatorService";+import {+ assertCompleteEvaluatorVariableMapping,+ extractEvaluatorPromptVariables,+} from "@/src/features/evals/v2/server/evaluators/evaluatorValidation";+import { prepareModernRuleVariableMapping } from "@/src/features/evals/v2/fns/variableMapping/prepareModernRuleVariableMapping";++export async function prepareBatchEvalEvaluatorMappings(params: {+ prisma: PrismaClient;+ projectId: string;+ mappings: BatchEvalEvaluatorMapping[];+}): Promise<BatchEvalEvaluatorMapping[]> {+ const { prisma, projectId, mappings } = params;+ const evaluatorIds = mappings.map(({ evaluatorId }) => evaluatorId);+ const evaluators = await findEvaluatorsByIds({+ prisma,+ projectId,+ evaluatorIds,+ });+ if (evaluators.length !== evaluatorIds.length) {+ throw new InvalidRequestError(+ "Selected evaluators are missing or incompatible with batch evaluation.",+ );+ }++ const evaluatorById = new Map(+ evaluators.map((evaluator) => [evaluator.id, evaluator]),+ );++ return mappings.map((mapping) => {+ const evaluator = evaluatorById.get(mapping.evaluatorId);+ if (!evaluator) {+ throw new InvalidRequestError(+ "Selected evaluators are missing or incompatible with batch evaluation.",+ );+ }+ try {+ const latestVersion = evaluator.versions[0];+ if (!latestVersion) {+ throw new InvalidRequestError("Evaluator version not found");+ }++ if (evaluator.type === EvalTemplateType.CODE) {+ if (mapping.variableMapping !== null) {+ throw new InvalidRequestError(+ "Code evaluator mappings are managed by Langfuse and cannot be provided.",+ );+ }+ return {+ evaluatorId: mapping.evaluatorId,+ variableMapping: null,+ };+ }++ const prepared = prepareModernRuleVariableMapping(+ latestVersion.variableMapping,+ evaluator.type,+ );+ const storedVariableMapping =+ mapping.variableMapping ?? prepared.initialVariableMapping;+ const promptMessages = reconcileEvaluatorPromptMessages({+ prompt: latestVersion.prompt,+ promptMessages: latestVersion.promptMessages,+ });+ assertCompleteEvaluatorVariableMapping({+ promptVariables: extractEvaluatorPromptVariables(promptMessages),+ variableMapping:+ storedVariableMapping ?? prepared.defaultVariableMapping,+ });++ return {+ evaluatorId: mapping.evaluatorId,+ variableMapping: storedVariableMapping,+ };+ } catch (error) {+ if (error instanceof InvalidRequestError) {+ throw new InvalidRequestError(+ `Evaluator "${evaluator.name}": ${error.message}`,+ );+ }+ throw error;+ }+ });+}diff --git a/web/src/features/batch-actions/server/runEvaluationRouter.ts b/web/src/features/batch-actions/server/runEvaluationRouter.tsindex d3578ee531..5e860abcc8 100644--- a/web/src/features/batch-actions/server/runEvaluationRouter.ts+++ b/web/src/features/batch-actions/server/runEvaluationRouter.ts@@ -23,6 +23,7 @@ import {import { env } from "@/src/env.mjs";import { CreateObservationBatchEvaluationActionSchema } from "../validation";import { batchEligibleEvaluatorWhere } from "@/src/features/evals/v2/server/evaluators/evaluatorRepository";+import { prepareBatchEvalEvaluatorMappings } from "./prepareBatchEvalEvaluatorMappings";export const runEvaluationRouter = createTRPCRouter({create: protectedProjectProcedure@@ -40,6 +41,7 @@ export const runEvaluationRouter = createTRPCRouter({query,evaluatorIds: rawEvaluatorIds,sourceTable = BatchEvalSourceTable.EVENTS,+ evaluatorMappings: rawEvaluatorMappings,} = input;if (env.LANGFUSE_MIGRATION_V4_ALLOW_PREVIEW_OPT_IN !== "true") {@@ -111,6 +113,15 @@ export const runEvaluationRouter = createTRPCRouter({});}+ const evaluatorMappings =+ input.evalVersion === "v2" && rawEvaluatorMappings+ ? await prepareBatchEvalEvaluatorMappings({+ prisma: ctx.prisma,+ projectId,+ mappings: rawEvaluatorMappings,+ })+ : undefined;+// Event comments live in Postgres, so resolve them for the preflight// count while retaining the original query for the queued worker.const commentFilterResult =@@ -145,6 +156,7 @@ export const runEvaluationRouter = createTRPCRouter({const batchConfig = {evaluatorIds,...(input.evalVersion ? { evalVersion: input.evalVersion } : {}),+ ...(evaluatorMappings ? { evaluatorMappings } : {}),};logger.info(@@ -193,6 +205,7 @@ export const runEvaluationRouter = createTRPCRouter({...(batchConfig.evalVersion? { evalVersion: batchConfig.evalVersion }: {}),+ ...(evaluatorMappings ? { evaluatorMappings } : {}),},},{diff --git a/web/src/features/batch-actions/validation.ts b/web/src/features/batch-actions/validation.tsindex 23d9d81ad3..96f8ce1b14 100644--- a/web/src/features/batch-actions/validation.ts+++ b/web/src/features/batch-actions/validation.ts@@ -2,22 +2,63 @@ import z from "zod";import {ObservationAddToDatasetConfigSchema,BatchActionQuerySchema,+ BatchEvalEvaluatorMappingSchema,BatchEvalSourceTableSchema,} from "@langfuse/shared";+/** Matches the evaluator options page size used by the run-evaluation dialog. */+export const BATCH_EVAL_EVALUATOR_LIMIT = 100;+export const CreateObservationAddToDatasetActionSchema = z.object({projectId: z.string(),query: BatchActionQuerySchema,config: ObservationAddToDatasetConfigSchema,});-export const CreateObservationBatchEvaluationActionSchema = z.object({- projectId: z.string(),- query: BatchActionQuerySchema,- evaluatorIds: z.array(z.string()).min(1),- sourceTable: BatchEvalSourceTableSchema.default("events"),- evalVersion: z.literal("v2").optional(),-});+export const CreateObservationBatchEvaluationActionSchema = z+ .object({+ projectId: z.string(),+ query: BatchActionQuerySchema,+ evaluatorIds: z.array(z.string()).min(1).max(BATCH_EVAL_EVALUATOR_LIMIT),+ sourceTable: BatchEvalSourceTableSchema.default("events"),+ evalVersion: z.literal("v2").optional(),+ evaluatorMappings: z+ .array(BatchEvalEvaluatorMappingSchema)+ .max(BATCH_EVAL_EVALUATOR_LIMIT)+ .optional(),+ })+ .superRefine((value, ctx) => {+ if (!value.evaluatorMappings) return;++ if (value.evalVersion !== "v2") {+ ctx.addIssue({+ code: "custom",+ message: "Variable mapping overrides require evaluator v2.",+ path: ["evaluatorMappings"],+ });+ return;+ }++ const evaluatorIds = new Set(value.evaluatorIds);+ const mappingIds = new Set<string>();+ for (const [index, mapping] of value.evaluatorMappings.entries()) {+ if (!evaluatorIds.has(mapping.evaluatorId)) {+ ctx.addIssue({+ code: "custom",+ message: "Mapping override must refer to a selected evaluator.",+ path: ["evaluatorMappings", index, "evaluatorId"],+ });+ }+ if (mappingIds.has(mapping.evaluatorId)) {+ ctx.addIssue({+ code: "custom",+ message: "An evaluator can only have one mapping override.",+ path: ["evaluatorMappings", index, "evaluatorId"],+ });+ }+ mappingIds.add(mapping.evaluatorId);+ }+ });export const GetBatchActionByIdSchema = z.object({projectId: z.string(),diff --git a/web/src/features/evals/v2/components/Rules/EvaluatorAssignmentsEditor/EvaluatorAssignmentsEditor.tsx b/web/src/features/evals/v2/components/Rules/EvaluatorAssignmentsEditor/EvaluatorAssignmentsEditor.tsxindex d7200c5765..d232d9ab89 100644--- a/web/src/features/evals/v2/components/Rules/EvaluatorAssignmentsEditor/EvaluatorAssignmentsEditor.tsx+++ b/web/src/features/evals/v2/components/Rules/EvaluatorAssignmentsEditor/EvaluatorAssignmentsEditor.tsx@@ -97,9 +97,9 @@ export function EvaluatorAssignmentsEditor({variant="ghost"size="sm"disabled={disabled}- className="text-foreground hover:text-foreground h-auto px-0 py-0 text-xs underline-offset-4 hover:bg-transparent hover:underline"+ className="text-foreground hover:text-foreground inline-flex h-auto items-center gap-1.5 px-0 py-0 text-xs leading-none underline-offset-4 hover:bg-transparent hover:underline">- <Plus className="mr-1.5 h-3.5 w-3.5" />+ <Plus className="size-3.5 shrink-0" aria-hidden="true" />Attach another evaluator</Button>)}@@ -146,6 +146,7 @@ export function EvaluatorAssignmentsEditor({defaultVariableMapping:evaluator.defaultVariableMapping,variableMapping: evaluator.initialVariableMapping,+ requiredVariables: evaluator.requiredVariables,});setPickerOpen(false);}}diff --git a/web/src/features/evals/v2/components/Rules/RuleSetup/components/RuleEvaluatorCostEstimate.tsx b/web/src/features/evals/v2/components/Rules/RuleSetup/components/RuleEvaluatorCostEstimate.tsxindex 7197a8a231..734581266a 100644--- a/web/src/features/evals/v2/components/Rules/RuleSetup/components/RuleEvaluatorCostEstimate.tsx+++ b/web/src/features/evals/v2/components/Rules/RuleSetup/components/RuleEvaluatorCostEstimate.tsx@@ -18,7 +18,9 @@ export function RuleEvaluatorCostEstimate({<span className="cursor-help font-mono text-sm tabular-nums underline decoration-dotted underline-offset-4">{estimate.estimatedCostUsd === null? "Unavailable"- : `≈ ${usdFormatter(estimate.estimatedCostUsd, 2, 2)} / week`}+ : estimate.period === "selection"+ ? `≈ ${usdFormatter(estimate.estimatedCostUsd, 2, 2)}`+ : `≈ ${usdFormatter(estimate.estimatedCostUsd, 2, 2)} / week`}</span></TooltipTrigger><TooltipContent className="max-w-72">diff --git a/web/src/features/evals/v2/fns/formatEvaluatorCostCalculation.clienttest.ts b/web/src/features/evals/v2/fns/formatEvaluatorCostCalculation.clienttest.tsindex 0ff266eedf..8d39a5b9eb 100644--- a/web/src/features/evals/v2/fns/formatEvaluatorCostCalculation.clienttest.ts+++ b/web/src/features/evals/v2/fns/formatEvaluatorCostCalculation.clienttest.ts@@ -59,4 +59,34 @@ describe("formatEvaluatorCostCalculation", () => {"Code evaluators do not call an LLM, so they do not incur model-provider / LLM costs.",);});++ it("does not claim a 7-day window when a selection has no priced test", () => {+ expect(+ formatEvaluatorCostCalculation({+ matchingObservations: 3,+ sampling: 1,+ testRunCostUsd: null,+ estimatedCostUsd: null,+ evaluatorType: EvalTemplateTypeEnum.LLM_AS_JUDGE,+ period: "selection",+ }),+ ).toBe(+ "No cost-bearing evaluator trace was available, and the fallback test call did not return a usable model cost. Expected cost would be charged to your linked API key, not Langfuse.",+ );+ });++ it("describes a one-shot selection instead of a weekly total", () => {+ expect(+ formatEvaluatorCostCalculation({+ matchingObservations: 12,+ sampling: 1,+ testRunCostUsd: 0.01,+ estimatedCostUsd: 0.12,+ evaluatorType: EvalTemplateTypeEnum.LLM_AS_JUDGE,+ period: "selection",+ }),+ ).toBe(+ "12 observations × 100% sampling × $0.01 per evaluation = ≈ $0.12 for this run. Expected cost on your linked API key (not Langfuse). Per-evaluation cost uses the latest cost-bearing evaluator trace from that period, or a fallback test call.",+ );+ });});diff --git a/web/src/features/evals/v2/fns/formatEvaluatorCostCalculation.ts b/web/src/features/evals/v2/fns/formatEvaluatorCostCalculation.tsindex a154c7d077..6a4f8faa19 100644--- a/web/src/features/evals/v2/fns/formatEvaluatorCostCalculation.ts+++ b/web/src/features/evals/v2/fns/formatEvaluatorCostCalculation.ts@@ -7,28 +7,36 @@ export function formatEvaluatorCostCalculation({testRunCostUsd,estimatedCostUsd,evaluatorType,+ period = "week",}: {matchingObservations: number;sampling: number;testRunCostUsd: number | null;estimatedCostUsd: number | null;evaluatorType: EvalTemplateType;+ period?: "week" | "selection";}) {if (evaluatorType === EvalTemplateTypeEnum.CODE) {return "Code evaluators do not call an LLM, so they do not incur model-provider / LLM costs.";}- const observations = `${numberFormatter(matchingObservations, 0)} matching observations`;+ const isSelection = period === "selection";+ const observations = isSelection+ ? `${numberFormatter(matchingObservations, 0)} observations`+ : `${numberFormatter(matchingObservations, 0)} matching observations`;const samplingRate = `${numberFormatter(sampling * 100, 0, 2)}% sampling`;- const scope =- "Expected cost on your linked API key (not Langfuse). Matching observations are based on the last 7 days.";+ const periodLabel = isSelection ? "for this run" : "/ week";+ const scope = isSelection+ ? "Expected cost on your linked API key (not Langfuse)."+ : "Expected cost on your linked API key (not Langfuse). Matching observations are based on the last 7 days.";if (matchingObservations === 0) {- return `${observations} × ${samplingRate} = ≈ ${usdFormatter(0, 2, 2)} / week. ${scope}`;+ return `${observations} × ${samplingRate} = ≈ ${usdFormatter(0, 2, 2)} ${periodLabel}. ${scope}`;}if (testRunCostUsd === null || estimatedCostUsd === null) {- return `No cost-bearing evaluator trace was available from the last 7 days, and the fallback test call did not return a usable model cost. Expected cost would be charged to your linked API key, not Langfuse.`;+ const window = isSelection ? "" : " from the last 7 days";+ return `No cost-bearing evaluator trace was available${window}, and the fallback test call did not return a usable model cost. Expected cost would be charged to your linked API key, not Langfuse.`;}- return `${observations} × ${samplingRate} × ${usdFormatter(testRunCostUsd, 2, 6)} per evaluation = ≈ ${usdFormatter(estimatedCostUsd, 2, 2)} / week. ${scope} Per-evaluation cost uses the latest cost-bearing evaluator trace from that period, or a fallback test call.`;+ return `${observations} × ${samplingRate} × ${usdFormatter(testRunCostUsd, 2, 6)} per evaluation = ≈ ${usdFormatter(estimatedCostUsd, 2, 2)} ${periodLabel}. ${scope} Per-evaluation cost uses the latest cost-bearing evaluator trace from that period, or a fallback test call.`;}diff --git a/web/src/features/evals/v2/fns/variableMapping/coverEvaluatorPromptVariables.ts b/web/src/features/evals/v2/fns/variableMapping/coverEvaluatorPromptVariables.tsnew file mode 100644index 0000000000..784bc59fc8--- /dev/null+++ b/web/src/features/evals/v2/fns/variableMapping/coverEvaluatorPromptVariables.ts@@ -0,0 +1,25 @@+import type { ObservationVariableMapping } from "@langfuse/shared";++/**+ * Adds empty mapping rows for prompt variables the stored mapping does not+ * already list, so the editor can collect a source column for each one.+ */+export function coverEvaluatorPromptVariables(+ mapping: ObservationVariableMapping[],+ promptVariables: string[],+): ObservationVariableMapping[] {+ const existing = new Set(mapping.map((entry) => entry.templateVariable));+ const missing = promptVariables.filter((variable) => !existing.has(variable));+ if (missing.length === 0) {+ return mapping;+ }++ return [+ ...mapping,+ ...missing.map((templateVariable) => ({+ templateVariable,+ selectedColumnId: "",+ jsonSelector: null,+ })),+ ];+}diff --git a/web/src/features/evals/v2/fns/variableMapping/prepareModernRuleVariableMapping.clienttest.ts b/web/src/features/evals/v2/fns/variableMapping/prepareModernRuleVariableMapping.clienttest.tsindex dddeaa6541..f8644f164d 100644--- a/web/src/features/evals/v2/fns/variableMapping/prepareModernRuleVariableMapping.clienttest.ts+++ b/web/src/features/evals/v2/fns/variableMapping/prepareModernRuleVariableMapping.clienttest.ts@@ -3,6 +3,7 @@ import {EvalTemplateType,} from "@langfuse/shared";import { describe, expect, it } from "vitest";+import { coverEvaluatorPromptVariables } from "./coverEvaluatorPromptVariables";import { prepareModernRuleVariableMapping } from "./prepareModernRuleVariableMapping";describe("prepareModernRuleVariableMapping", () => {@@ -100,3 +101,25 @@ describe("prepareModernRuleVariableMapping", () => {});});});++describe("coverEvaluatorPromptVariables", () => {+ it("adds empty rows for prompt variables missing from the stored mapping", () => {+ expect(+ coverEvaluatorPromptVariables(+ [{ templateVariable: "output", selectedColumnId: "output" }],+ ["input", "output"],+ ),+ ).toEqual([+ { templateVariable: "output", selectedColumnId: "output" },+ { templateVariable: "input", selectedColumnId: "", jsonSelector: null },+ ]);+ });++ it("returns the original mapping when every prompt variable is already listed", () => {+ const mapping = [+ { templateVariable: "output", selectedColumnId: "output" },+ ];++ expect(coverEvaluatorPromptVariables(mapping, ["output"])).toBe(mapping);+ });+});diff --git a/web/src/features/evals/v2/hooks/useRuleCostEstimate.ts b/web/src/features/evals/v2/hooks/useRuleCostEstimate.tsindex 8193fdf264..a60d94eb7f 100644--- a/web/src/features/evals/v2/hooks/useRuleCostEstimate.ts+++ b/web/src/features/evals/v2/hooks/useRuleCostEstimate.ts@@ -14,6 +14,7 @@ export type RuleCostEstimate = {testRunCostUsd: number | null;estimatedCostUsd: number | null;evaluatorType: EvalTemplateType;+ period?: "week" | "selection";};const ESTIMATE_DEBOUNCE_MS = 500;@@ -27,9 +28,11 @@ type EstimateState = {export function useRuleCostEstimate({projectId,store,+ matchingObservations: matchingObservationsOverride,}: {projectId: string;store: RuleSetupStore;+ matchingObservations?: number;}) {const { filter, sampling, assignments } = useStore(store,@@ -163,7 +166,7 @@ export function useRuleCostEstimate({);updateState({status: "idle",- estimates: assignments.flatMap((assignment) => {+ estimates: assignments.flatMap((assignment): RuleCostEstimate[] => {if (assignment.evaluatorType === EvalTemplateType.CODE) {return codeEstimates.filter(({ evaluatorId }) => evaluatorId === assignment.evaluatorId,@@ -192,17 +195,24 @@ export function useRuleCostEstimate({return {...state,- estimates: state.estimates.map((estimate) => ({- ...estimate,- sampling,- estimatedCostUsd:- estimate.matchingObservations === 0- ? 0- : estimate.testRunCostUsd === null- ? null- : estimate.matchingObservations *- sampling *- estimate.testRunCostUsd,- })),+ estimates: state.estimates.map((estimate) => {+ const matchingObservations =+ matchingObservationsOverride ?? estimate.matchingObservations;+ return {+ ...estimate,+ matchingObservations,+ sampling,+ period:+ matchingObservationsOverride === undefined+ ? ("week" as const)+ : ("selection" as const),+ estimatedCostUsd:+ matchingObservations === 0+ ? 0+ : estimate.testRunCostUsd === null+ ? null+ : matchingObservations * sampling * estimate.testRunCostUsd,+ };+ }),};}diff --git a/web/src/features/evals/v2/types/rules.ts b/web/src/features/evals/v2/types/rules.tsindex e107e12180..abc56cf28e 100644--- a/web/src/features/evals/v2/types/rules.ts+++ b/web/src/features/evals/v2/types/rules.ts@@ -14,6 +14,7 @@ type RuleDraftAssignment = {evaluatorType: EvalTemplateType;defaultVariableMapping: ObservationVariableMapping[];variableMapping: ObservationVariableMapping[] | null;+ requiredVariables?: string[];};export type RuleDraft = {@@ -31,6 +32,7 @@ export type RuleEvaluatorOption = {createdByUser?: { name: string | null; email: string | null } | null;defaultVariableMapping: ObservationVariableMapping[];initialVariableMapping: ObservationVariableMapping[] | null;+ requiredVariables?: string[];};export type ActivationConfirmationRequest = {diff --git a/web/src/features/posthog-analytics/usePostHogClientCapture.ts b/web/src/features/posthog-analytics/usePostHogClientCapture.tsindex 16e9a424f0..8e24764d6a 100644--- a/web/src/features/posthog-analytics/usePostHogClientCapture.ts+++ b/web/src/features/posthog-analytics/usePostHogClientCapture.ts@@ -196,6 +196,9 @@ const events = {"detach_evaluator","filter_reused",],+ // One-shot batch evaluation from the events / experiments tables.+ // Counts and enums only — never mapping contents or observation payloads.+ batch_eval: ["run"],integrations: ["posthog_form_submitted","blob_storage_form_submitted",diff --git a/worker/src/__tests__/handleBatchActionJob-comment-filters.unit.test.ts b/worker/src/__tests__/handleBatchActionJob-comment-filters.unit.test.tsindex e92cefddf9..97fdaf8c4b 100644--- a/worker/src/__tests__/handleBatchActionJob-comment-filters.unit.test.ts+++ b/worker/src/__tests__/handleBatchActionJob-comment-filters.unit.test.ts@@ -242,4 +242,47 @@ describe("event batch-action comment filter wiring", () => {}),);});++ it("applies mapping overrides to ruleless in-memory assignments", async () => {+ const mapping = [{ templateVariable: "output", selectedColumnId: "input" }];+ mocks.findEvaluators.mockResolvedValue([+ {+ id: "standalone-evaluator",+ name: "Standalone evaluator",+ projectId: "project-1",+ type: "LLM_AS_JUDGE",+ blockedAt: null,+ versions: [{ id: "version-2", variableMapping: [] }],+ assignments: [],+ },+ ]);++ await runBatchAction({+ ...createPayload("observation-run-batched-evaluation"),+ evaluatorIds: ["standalone-evaluator"],+ evalVersion: "v2",+ evaluatorMappings: [+ {+ evaluatorId: "standalone-evaluator",+ variableMapping: mapping,+ },+ ],+ });++ expect(mocks.processBatchedObservationEval).toHaveBeenCalledWith(+ expect.objectContaining({+ evaluators: [+ expect.objectContaining({+ ruleId: null,+ assignments: [+ expect.objectContaining({+ evaluatorId: "standalone-evaluator",+ variableMapping: mapping,+ }),+ ],+ }),+ ],+ }),+ );+ });});diff --git a/worker/src/features/batchAction/handleBatchActionJob.ts b/worker/src/features/batchAction/handleBatchActionJob.tsindex addc66d4b2..317a45ccc8 100644--- a/worker/src/features/batchAction/handleBatchActionJob.ts+++ b/worker/src/features/batchAction/handleBatchActionJob.ts@@ -476,6 +476,7 @@ export const handleBatchActionJob = async (evaluatorIds,batchActionId,evalVersion,+ evaluatorMappings,} = batchActionEvent;if (!batchActionId) {@@ -514,6 +515,12 @@ export const handleBatchActionJob = async (});evaluatorLabels = stableEvaluators.map(({ name }) => name);+ const mappingByEvaluatorId = new Map(+ (evaluatorMappings ?? []).map((mapping) => [+ mapping.evaluatorId,+ mapping.variableMapping,+ ]),+ );// A batch run addresses the evaluator directly (`ruleId` stays null),// but uses a deterministic associated rule as its legacy execution// anchor so existing readers can still find it.@@ -531,7 +538,7 @@ export const handleBatchActionJob = async ({id: evaluator.id,evaluatorId: evaluator.id,- variableMapping: null,+ variableMapping: mappingByEvaluatorId.get(evaluator.id) ?? null,evaluator: {id: evaluator.id,projectId: evaluator.projectId,diff --git a/worker/src/features/batchAction/processBatchedObservationEval.test.ts b/worker/src/features/batchAction/processBatchedObservationEval.test.tsindex 9746a744bd..790d4a5929 100644--- a/worker/src/features/batchAction/processBatchedObservationEval.test.ts+++ b/worker/src/features/batchAction/processBatchedObservationEval.test.ts@@ -84,6 +84,7 @@ describe("processBatchedObservationEval", () => {expect.objectContaining({configs: evaluators,executionMode: "MANUAL",+ executionScopeId: batchActionId,}),);expect(@@ -150,6 +151,7 @@ describe("processBatchedObservationEval", () => {expect.objectContaining({configs: evaluators,executionMode: "MANUAL",+ executionScopeId: "batch-action-v2",}),);});diff --git a/worker/src/features/batchAction/processBatchedObservationEval.ts b/worker/src/features/batchAction/processBatchedObservationEval.tsindex 390d776110..791f31e2ec 100644--- a/worker/src/features/batchAction/processBatchedObservationEval.ts+++ b/worker/src/features/batchAction/processBatchedObservationEval.ts@@ -63,6 +63,7 @@ export async function processBatchedObservationEval(params: {configs: evaluators,schedulerDeps,executionMode: "MANUAL",+ executionScopeId: batchActionId,});}),),diff --git a/worker/src/features/evaluation/observationEval/__tests__/createSchedulerDeps.test.ts b/worker/src/features/evaluation/observationEval/__tests__/createSchedulerDeps.test.tsindex 7975f0a593..1dc0b4cbf8 100644--- a/worker/src/features/evaluation/observationEval/__tests__/createSchedulerDeps.test.ts+++ b/worker/src/features/evaluation/observationEval/__tests__/createSchedulerDeps.test.ts@@ -78,4 +78,33 @@ describe("createObservationEvalSchedulerDeps", () => {);expect(getLLMQueueInstance).not.toHaveBeenCalled();});++ it("includes a mapping override on the observation eval payload", async () => {+ const { createObservationEvalSchedulerDeps } =+ await import("../createSchedulerDeps");+ const variableMapping = [+ { templateVariable: "output", selectedColumnId: "input" },+ ];++ await createObservationEvalSchedulerDeps().enqueueEvalJob({+ projectId: "project-1",+ jobExecutionId: "job-3",+ observationS3Path: "evals/project-1/observations/obs-1.json",+ delay: 0,+ evalTemplateType: EvalTemplateType.LLM_AS_JUDGE,+ evaluatorId: "evaluator-1",+ variableMapping,+ });++ expect(addToLLMQueue).toHaveBeenCalledWith(+ "llm-as-a-judge-execution-queue",+ expect.objectContaining({+ payload: expect.objectContaining({+ evaluatorId: "evaluator-1",+ variableMapping,+ }),+ }),+ { delay: 0 },+ );+ });});diff --git a/worker/src/features/evaluation/observationEval/__tests__/observationEvalProcessor.test.ts b/worker/src/features/evaluation/observationEval/__tests__/observationEvalProcessor.test.tsindex 5136d8d892..f6070cbad3 100644--- a/worker/src/features/evaluation/observationEval/__tests__/observationEvalProcessor.test.ts+++ b/worker/src/features/evaluation/observationEval/__tests__/observationEvalProcessor.test.ts@@ -508,6 +508,44 @@ describe("processObservationEval", () => {);});+ it("uses a queue mapping override for a batch-run evaluator without a rule", async () => {+ const variableMapping = [+ { templateVariable: "output", selectedColumnId: "input" },+ ];+ (prisma.jobExecution.findFirst as Mock).mockResolvedValue(+ createMockJobExecution({+ id: jobExecutionId,+ projectId,+ jobConfigurationId: evaluator.id,+ jobTemplateId: null,+ }),+ );+ (prisma.evaluator.findFirst as Mock).mockResolvedValue(evaluator);++ await processObservationEval({+ event: {+ ...baseEvent,+ executionMode: "MANUAL",+ evaluatorId: evaluator.id,+ variableMapping,+ },+ executionType: EvalTemplateType.LLM_AS_JUDGE,+ deps: createMockProcessorDeps(),+ });++ expect(+ prisma.evaluationRuleEvaluatorAssignment.findFirst,+ ).not.toHaveBeenCalled();+ expect(runLLMAsJudgeEvaluation).toHaveBeenCalledWith(+ expect.objectContaining({+ config: expect.objectContaining({ variableMapping }),+ executionMetadata: expect.not.objectContaining({+ evaluation_rule_assignment_id: expect.anything(),+ }),+ }),+ );+ });+it("uses an empty variable mapping when neither v2 mapping is configured", async () => {setupV2Job();(diff --git a/worker/src/features/evaluation/observationEval/__tests__/scheduleObservationEvals.test.ts b/worker/src/features/evaluation/observationEval/__tests__/scheduleObservationEvals.test.tsindex b9a642248b..9c4f652867 100644--- a/worker/src/features/evaluation/observationEval/__tests__/scheduleObservationEvals.test.ts+++ b/worker/src/features/evaluation/observationEval/__tests__/scheduleObservationEvals.test.ts@@ -637,6 +637,173 @@ describe("scheduleObservationEvals", () => {expect(payload).not.toHaveProperty("evaluationRuleId");});+ it("should carry a mapping override for ruleless batch runs", async () => {+ const schedulerDeps = createMockSchedulerDeps();+ const variableMapping = [+ { templateVariable: "output", selectedColumnId: "input" },+ ];++ await scheduleObservationEvals({+ observation: createMockObservation(),+ configs: [+ createMockRule({+ id: "evaluator-1",+ ruleId: null,+ assignments: [+ {+ id: "evaluator-1",+ evaluatorId: "evaluator-1",+ variableMapping,+ evaluator: {+ id: "evaluator-1",+ projectId: "project-789",+ type: EvalTemplateType.LLM_AS_JUDGE,+ },+ },+ ],+ }),+ ],+ schedulerDeps,+ executionMode: "MANUAL",+ });++ expect(schedulerDeps.enqueueEvalJob).toHaveBeenCalledWith(+ expect.objectContaining({+ evaluatorId: "evaluator-1",+ variableMapping,+ }),+ );+ });++ it("should include an execution scope in the job identity", async () => {+ const schedulerDeps = createMockSchedulerDeps();+ const observation = createMockObservation();+ const rule = createMockRule({+ id: "evaluator-1",+ ruleId: null,+ assignments: [+ {+ id: "evaluator-1",+ evaluatorId: "evaluator-1",+ variableMapping: null,+ evaluator: {+ id: "evaluator-1",+ projectId: "project-789",+ type: EvalTemplateType.LLM_AS_JUDGE,+ },+ },+ ],+ });++ await scheduleObservationEvals({+ observation,+ configs: [rule],+ schedulerDeps,+ executionMode: "MANUAL",+ executionScopeId: "batch-action-1",+ });++ const scopedJobExecutionId = createW3CTraceId(+ JSON.stringify([+ "observation-eval",+ "evaluator-1",+ "evaluator-1",+ observation.trace_id,+ observation.span_id,+ "batch-action-1",+ ]),+ );+ const unscopedJobExecutionId = createW3CTraceId(+ JSON.stringify([+ "observation-eval",+ "evaluator-1",+ "evaluator-1",+ observation.trace_id,+ observation.span_id,+ ]),+ );++ expect(scopedJobExecutionId).not.toBe(unscopedJobExecutionId);+ expect(schedulerDeps.upsertJobExecution).toHaveBeenCalledWith(+ expect.objectContaining({ id: scopedJobExecutionId }),+ );+ });++ it("should keep live eval job identity unchanged when no execution scope is passed", async () => {+ const schedulerDeps = createMockSchedulerDeps();+ const observation = createMockObservation();+ const rule = createMockRule();++ await scheduleObservationEvals({+ observation,+ configs: [rule],+ schedulerDeps,+ });++ expect(schedulerDeps.upsertJobExecution).toHaveBeenCalledWith(+ expect.objectContaining({+ id: createW3CTraceId(+ JSON.stringify([+ "observation-eval",+ rule.id,+ "assignment-1",+ observation.trace_id,+ observation.span_id,+ ]),+ ),+ }),+ );+ });++ it("should omit mapping when a ruleless batch run inherits the version mapping", async () => {+ const schedulerDeps = createMockSchedulerDeps();++ await scheduleObservationEvals({+ observation: createMockObservation(),+ configs: [createMockRule({ id: "evaluator-1", ruleId: null })],+ schedulerDeps,+ executionMode: "MANUAL",+ });++ const payload = vi.mocked(schedulerDeps.enqueueEvalJob).mock.calls[0]![0];+ expect(payload).toMatchObject({ evaluatorId: "evaluator-1" });+ expect(payload).not.toHaveProperty("variableMapping");+ });++ it("should omit mapping for rule-backed assignments", async () => {+ const schedulerDeps = createMockSchedulerDeps();++ await scheduleObservationEvals({+ observation: createMockObservation(),+ configs: [+ createMockRule({+ assignments: [+ {+ id: "assignment-1",+ evaluatorId: "evaluator-1",+ variableMapping: [+ { templateVariable: "output", selectedColumnId: "input" },+ ],+ evaluator: {+ id: "evaluator-1",+ projectId: "project-789",+ type: EvalTemplateType.LLM_AS_JUDGE,+ },+ },+ ],+ }),+ ],+ schedulerDeps,+ });++ const payload = vi.mocked(schedulerDeps.enqueueEvalJob).mock.calls[0]![0];+ expect(payload).toMatchObject({+ evaluatorId: "evaluator-1",+ evaluationRuleId: "rule-1",+ });+ expect(payload).not.toHaveProperty("variableMapping");+ });+it("should not carry evaluator identity for legacy configs", async () => {const schedulerDeps = createMockSchedulerDeps();diff --git a/worker/src/features/evaluation/observationEval/createSchedulerDeps.ts b/worker/src/features/evaluation/observationEval/createSchedulerDeps.tsindex c4b0c1d923..c9f3904629 100644--- a/worker/src/features/evaluation/observationEval/createSchedulerDeps.ts+++ b/worker/src/features/evaluation/observationEval/createSchedulerDeps.ts@@ -82,6 +82,9 @@ export function createObservationEvalSchedulerDeps(): ObservationEvalSchedulerDe...(params.evaluationRuleId? { evaluationRuleId: params.evaluationRuleId }: {}),+ ...(params.variableMapping != null+ ? { variableMapping: params.variableMapping }+ : {}),};if (params.evalTemplateType === EvalTemplateType.CODE) {diff --git a/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts b/worker/src/features/evaluation/observationEval/observationEvalProcessor.tsindex 81908385b0..6e8ac66e9f 100644--- a/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts+++ b/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts@@ -358,9 +358,16 @@ async function resolveObservationEvalExecution(params: {where: { id: evaluatorId, projectId, type: executionType },include: evaluatorInclude,});- return evaluator- ? buildV2Execution({ rule: null, assignment: null, evaluator })- : { type: "cancelled" as const, reason: "evaluator-unavailable" };+ if (!evaluator) {+ return { type: "cancelled" as const, reason: "evaluator-unavailable" };+ }+ // Ruleless batch runs have no assignment row. A mapping on the queue+ // payload is the override; omitting it inherits the version mapping.+ const assignment =+ event.variableMapping !== undefined+ ? { variableMapping: event.variableMapping }+ : null;+ return buildV2Execution({ rule: null, assignment, evaluator });}// The assignment is what authorizes this execution — it is the (rule,@@ -460,10 +467,10 @@ type ResolvedEvaluationRule = Prisma.EvaluationRuleGetPayload<object>;*/function buildV2Execution(params: {rule: ResolvedEvaluationRule | null;- assignment: Pick<- Prisma.EvaluationRuleEvaluatorAssignmentGetPayload<object>,- "id" | "variableMapping"- > | null;+ assignment: {+ id?: string;+ variableMapping: Prisma.JsonValue;+ } | null;evaluator: ResolvedEvaluator;}) {const { rule, assignment, evaluator } = params;diff --git a/worker/src/features/evaluation/observationEval/scheduleObservationEvals.ts b/worker/src/features/evaluation/observationEval/scheduleObservationEvals.tsindex a3c8f025cc..8ec12af32e 100644--- a/worker/src/features/evaluation/observationEval/scheduleObservationEvals.ts+++ b/worker/src/features/evaluation/observationEval/scheduleObservationEvals.ts@@ -1,3 +1,4 @@+import type { ObservationVariableMapping } from "@langfuse/shared";import {type ObservationForEval,type ObservationEvalAssignment,@@ -19,6 +20,7 @@ import {type EvalExecutionMode,canRunEvalRule,mapEventEvalFilterColumnIdToField,+ observationVariableMappingList,} from "@langfuse/shared";import { createW3CTraceId } from "../../utils";import { isInternalEvalEnvironment } from "../isEvalTargetEnvironmentAllowed";@@ -28,6 +30,13 @@ interface ScheduleObservationEvalsParams {configs: ObservationEvalRule[];schedulerDeps: ObservationEvalSchedulerDeps;executionMode?: EvalExecutionMode;+ /**+ * Extra identity for this scheduling pass so overlapping runs of the same+ * evaluator on the same observation do not share a job id. Live ingestion+ * omits it; batch passes the batch-action id so retries of that run stay+ * stable while a second run gets its own executions.+ */+ executionScopeId?: string;}/**@@ -84,7 +93,13 @@ export function isObservationAllowedForQueuedObservationEvals(export async function scheduleObservationEvals(params: ScheduleObservationEvalsParams,): Promise<void> {- const { observation, configs, schedulerDeps, executionMode } = params;+ const {+ observation,+ configs,+ schedulerDeps,+ executionMode,+ executionScopeId,+ } = params;// Early return if no configsif (configs.length === 0) {@@ -167,6 +182,7 @@ export async function scheduleObservationEvals(observationS3Path,schedulerDeps,executionMode,+ executionScopeId,}).catch((error) => {logger.error("Failed to process observation eval assignment", {configId: config.id,@@ -188,6 +204,7 @@ interface ProcessConfigParams {observationS3Path: string;schedulerDeps: ObservationEvalSchedulerDeps;executionMode?: EvalExecutionMode;+ executionScopeId?: string;}async function processMatchingConfig(@@ -200,26 +217,28 @@ async function processMatchingConfig(observationS3Path,schedulerDeps,executionMode,+ executionScopeId,} = params;- const jobExecutionId = createW3CTraceId(- JSON.stringify(- "assignments" in matchingConfig- ? [- "observation-eval",- matchingConfig.id,- assignment.id,- observation.trace_id,- observation.span_id,- ]- : [- "observation-eval",- matchingConfig.id,- observation.trace_id,- observation.span_id,- ],- ),- );+ const jobIdentity: string[] =+ "assignments" in matchingConfig+ ? [+ "observation-eval",+ matchingConfig.id,+ assignment.id,+ observation.trace_id,+ observation.span_id,+ ]+ : [+ "observation-eval",+ matchingConfig.id,+ observation.trace_id,+ observation.span_id,+ ];+ if (executionScopeId) {+ jobIdentity.push(executionScopeId);+ }+ const jobExecutionId = createW3CTraceId(JSON.stringify(jobIdentity));// Create job executionawait schedulerDeps.upsertJobExecution({@@ -252,6 +271,9 @@ async function processMatchingConfig(: {}),}: {}),+ ...(assignment.variableMapping != null+ ? { variableMapping: assignment.variableMapping }+ : {}),});logger.debug("Scheduled observation eval job", {@@ -269,6 +291,12 @@ type ScheduledObservationEvalAssignment = {/** Legacy template id, or evaluator id for a ruleless V2 batch run. */evalTemplateId: string | null;evaluatorType: ObservationEvalAssignment["evaluator"]["type"];+ /**+ * Mapping override for a ruleless batch run. Omitted when the run should+ * inherit the evaluator version mapping, and never set for rule-backed jobs+ * (those load the assignment row at pickup).+ */+ variableMapping?: ObservationVariableMapping[];};function getExecutableAssignments(@@ -300,6 +328,10 @@ function getExecutableAssignments(return [];}+ const parsedMapping = observationVariableMappingList.safeParse(+ assignment.variableMapping,+ );+return [{id: assignment.id,@@ -308,6 +340,9 @@ function getExecutableAssignments(// Ruleless batch runs use the evaluator as the legacy template anchor.evalTemplateId: rule.ruleId === null ? assignment.evaluator.id : null,evaluatorType: assignment.evaluator.type,+ ...(rule.ruleId === null && parsedMapping.success+ ? { variableMapping: parsedMapping.data }+ : {}),},];});diff --git a/worker/src/features/evaluation/observationEval/types.ts b/worker/src/features/evaluation/observationEval/types.tsindex 06bcf4b35e..e156251c1e 100644--- a/worker/src/features/evaluation/observationEval/types.ts+++ b/worker/src/features/evaluation/observationEval/types.ts@@ -8,7 +8,11 @@ import {JobExecutionStatus,} from "@langfuse/shared/src/db";import { type EvalExecutionMode } from "@langfuse/shared";-import type { EvalTargetObject, FilterState } from "@langfuse/shared";+import type {+ EvalTargetObject,+ FilterState,+ ObservationVariableMapping,+} from "@langfuse/shared";/*** Re-export ObservationForEval as the canonical observation type for eval operations.@@ -114,5 +118,10 @@ export interface ObservationEvalSchedulerDeps {/** Evaluator v2 identity; omitted when scheduling a legacy config. */evaluatorId?: string;evaluationRuleId?: string;+ /**+ * Mapping override for a ruleless batch run. Omitted to inherit the+ * evaluator version mapping.+ */+ variableMapping?: ObservationVariableMapping[];}) => Promise<void>;}