🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(evals): runBatchEvaluation MCP tool and batch_action_id on batch run outputs (#19314)
Tobias Wochingerf3500cd1167e
diff --git a/packages/shared/src/features/evals/evalExecutionMetadata.ts b/packages/shared/src/features/evals/evalExecutionMetadata.tsindex 2d350c1fbb..aaa383653b 100644--- a/packages/shared/src/features/evals/evalExecutionMetadata.ts+++ b/packages/shared/src/features/evals/evalExecutionMetadata.ts@@ -9,6 +9,7 @@ export const EvalExecutionMetadataKey = {TARGET_TRACE_ID: "target_trace_id",TARGET_OBSERVATION_ID: "target_observation_id",TARGET_DATASET_ITEM_ID: "target_dataset_item_id",+ BATCH_ACTION_ID: "batch_action_id",} as const;export type EvalExecutionMetadataKey =diff --git a/packages/shared/src/in-app-agent/server/mcpPolicy.ts b/packages/shared/src/in-app-agent/server/mcpPolicy.tsindex 7e70d3fd16..0b71caef96 100644--- a/packages/shared/src/in-app-agent/server/mcpPolicy.ts+++ b/packages/shared/src/in-app-agent/server/mcpPolicy.ts@@ -193,6 +193,10 @@ export const IN_APP_AGENT_LANGFUSE_MCP_TOOL_POLICIES = {approval: "approval",availability: { scope: "evaluationRule:CUD" },},+ runBatchEvaluation: {+ approval: "approval",+ availability: { scope: "evaluationRule:CUD" },+ },listExperiments: {approval: "auto",availability: { scope: "promptExperiments:read" },diff --git a/packages/shared/src/server/evals/evalExecutionMetadata.ts b/packages/shared/src/server/evals/evalExecutionMetadata.tsindex 7ccc18c685..f5105e71bc 100644--- a/packages/shared/src/server/evals/evalExecutionMetadata.ts+++ b/packages/shared/src/server/evals/evalExecutionMetadata.ts@@ -22,6 +22,8 @@ export function buildEvalExecutionData(targetTraceId: string | null;targetObservationId: string | null;targetDatasetItemId: string | null;+ /** Set when the execution belongs to a manual batch evaluation run. */+ batchActionId?: string;},) {// Keep old workers and legacy trace/observation sinks compatible during the@@ -59,6 +61,7 @@ export function buildEvalExecutionData(params.targetObservationId,[EvalExecutionMetadataKey.TARGET_DATASET_ITEM_ID]:params.targetDatasetItemId,+ [EvalExecutionMetadataKey.BATCH_ACTION_ID]: params.batchActionId,};const evaluationContext: EvalExecutionContext =diff --git a/packages/shared/src/server/queues.ts b/packages/shared/src/server/queues.tsindex 128421a447..2e3894aa93 100644--- a/packages/shared/src/server/queues.ts+++ b/packages/shared/src/server/queues.ts@@ -176,6 +176,9 @@ export const ObservationEvalExecutionEventSchema = z.object({// override. Optional for jobs queued before this field existed; those// inherit the evaluator version mapping.variableMapping: observationVariableMappingList.optional(),+ // Manual batch runs record their batch action on the execution trace and+ // scores so a run's outputs can be found together.+ batchActionId: z.string().optional(),});export const PostHogIntegrationProcessingEventSchema = z.object({projectId: z.string(),diff --git a/web/src/__tests__/server/mcp-tools-write.servertest.ts b/web/src/__tests__/server/mcp-tools-write.servertest.tsindex ba543c7643..98f260d082 100644--- a/web/src/__tests__/server/mcp-tools-write.servertest.ts+++ b/web/src/__tests__/server/mcp-tools-write.servertest.ts@@ -1,8 +1,15 @@+const queueMocks = vi.hoisted(() => ({+ batchActionQueueAdd: vi.fn(),+}));+// Mock queue operations to avoid Redis dependency in testsvi.mock("@langfuse/shared/src/server", async () => {const actual = await vi.importActual("@langfuse/shared/src/server");return {...actual,+ BatchActionQueue: {+ getInstance: () => ({ add: queueMocks.batchActionQueueAdd }),+ },// Mock queue getInstance to return a no-op queueEventPropagationQueue: {getInstance: () => ({@@ -109,6 +116,11 @@ import {import { externalMediaStorageFeature } from "@/src/features/mcp/server/externalMediaStorage";import { evalsFeature } from "@/src/features/mcp/server/evals";import { handleGetEvaluationRule } from "@/src/features/mcp/server/evals/tools/getEvaluationRule";+import {+ handleRunBatchEvaluation,+ runBatchEvaluationTool,+} from "@/src/features/mcp/server/evals/tools/runBatchEvaluation";+import { env } from "@/src/env.mjs";import { EvaluatorService } from "@/src/features/evals/v2/server/evaluators/evaluatorService";import {attachEvaluatorToEvaluationRuleTool,@@ -1083,6 +1095,153 @@ describe("MCP Write Tools", () => {});});+ describe("runBatchEvaluation tool", () => {+ const mutableEnv = env as unknown as {+ LANGFUSE_MIGRATION_V4_ALLOW_PREVIEW_OPT_IN: "true" | "false";+ };+ const originalPreviewOptIn =+ mutableEnv.LANGFUSE_MIGRATION_V4_ALLOW_PREVIEW_OPT_IN;++ beforeEach(() => {+ mutableEnv.LANGFUSE_MIGRATION_V4_ALLOW_PREVIEW_OPT_IN = "true";+ queueMocks.batchActionQueueAdd.mockReset();+ });++ afterAll(() => {+ mutableEnv.LANGFUSE_MIGRATION_V4_ALLOW_PREVIEW_OPT_IN =+ originalPreviewOptIn;+ });++ it("queues an evaluator v2 batch action attributed to the API key", async () => {+ verifyToolAnnotations(runBatchEvaluationTool, { expensiveHint: true });+ const setup = await createMcpTestSetup();+ const { projectId, apiKeyId } = setup;+ const evaluator = await createStableLlmEvaluatorForMcpWriteTest(setup);++ const { id } = (await handleRunBatchEvaluation(+ {+ evaluatorIds: [evaluator.id],+ filter: [{ column: "tags", operator: "all of", value: ["prod"] }],+ sampling: 0.5,+ rowLimit: 100,+ } as never,+ setup.context,+ )) as { id: string };++ const query = {+ filter: [+ {+ column: "traceTags",+ type: "arrayOptions",+ operator: "all of",+ value: ["prod"],+ },+ ],+ orderBy: null,+ };+ await expect(+ prisma.batchAction.findUniqueOrThrow({ where: { id, projectId } }),+ ).resolves.toMatchObject({+ userId: apiKeyId,+ actionType: "observation-run-batched-evaluation",+ status: "QUEUED",+ query,+ config: {+ evaluatorIds: [evaluator.id],+ evalVersion: "v2",+ sampling: 0.5,+ rowLimit: 100,+ },+ });+ expect(queueMocks.batchActionQueueAdd).toHaveBeenCalledWith(+ expect.anything(),+ expect.objectContaining({+ payload: expect.objectContaining({+ batchActionId: id,+ projectId,+ query,+ evaluatorIds: [evaluator.id],+ evalVersion: "v2",+ }),+ }),+ { jobId: id },+ );+ await expect(+ prisma.auditLog.findFirst({+ where: {+ projectId,+ apiKeyId,+ resourceType: "batchAction",+ resourceId: id,+ },+ }),+ ).resolves.toMatchObject({+ action: "observation-run-batched-evaluation",+ });+ });++ it("bounds the selection by the time range and the plan's data-access window", async () => {+ const setup = await createMcpTestSetup();+ const evaluator = await createStableLlmEvaluatorForMcpWriteTest(setup);+ const fromStartTime = "2020-01-01T00:00:00.000Z";+ const toStartTime = "2020-01-15T00:00:00.000Z";++ const { id } = (await handleRunBatchEvaluation(+ { evaluatorIds: [evaluator.id], fromStartTime, toStartTime } as never,+ { ...setup.context, plan: "cloud:hobby" },+ )) as { id: string };++ const { query } = await prisma.batchAction.findUniqueOrThrow({+ where: { id, projectId: setup.projectId },+ });+ const filter = (query as { filter: Array<Record<string, unknown>> })+ .filter;+ expect(filter.slice(0, 2)).toEqual([+ {+ type: "datetime",+ column: "startTime",+ operator: ">=",+ value: fromStartTime,+ },+ {+ type: "datetime",+ column: "startTime",+ operator: "<",+ value: toStartTime,+ },+ ]);+ const accessFloor = filter[2];+ expect(accessFloor).toMatchObject({+ type: "datetime",+ column: "startTime",+ operator: ">=",+ });+ const floorAgeDays =+ (Date.now() - new Date(accessFloor.value as string).getTime()) /+ (24 * 60 * 60 * 1000);+ expect(floorAgeDays).toBeCloseTo(30, 1);+ });++ it("rejects full-text filters, which batch queries cannot carry", async () => {+ const setup = await createMcpTestSetup();+ const evaluator = await createStableLlmEvaluatorForMcpWriteTest(setup);++ await expect(+ handleRunBatchEvaluation(+ {+ evaluatorIds: [evaluator.id],+ filter: [{ column: "input", operator: "matches", value: "refund" }],+ } as never,+ setup.context,+ ),+ ).rejects.toMatchObject({+ code: ErrorCode.InvalidParams,+ message: expect.stringContaining("filter.0.operator"),+ });+ expect(queueMocks.batchActionQueueAdd).not.toHaveBeenCalled();+ });+ });+describe("deleteEvaluator tool", () => {it("should have destructiveHint annotation", () => {verifyToolAnnotations(deleteEvaluatorTool, {diff --git a/web/src/__tests__/server/unit/batch-actions-comment-filter.servertest.ts b/web/src/__tests__/server/unit/batch-actions-comment-filter.servertest.tsindex b1047568f9..d584647b28 100644--- a/web/src/__tests__/server/unit/batch-actions-comment-filter.servertest.ts+++ b/web/src/__tests__/server/unit/batch-actions-comment-filter.servertest.ts@@ -98,6 +98,13 @@ function prepare({foundEvaluatorIds = [evaluatorId],missingPromptVariable = false,decisionModel = false,+ plan,+}: {+ v4BetaEnabled?: boolean;+ foundEvaluatorIds?: string[];+ missingPromptVariable?: boolean;+ decisionModel?: boolean;+ plan?: string;} = {}) {const batchActionCreate = vi.fn()@@ -161,7 +168,14 @@ function prepare({...createInnerTRPCContext({session: {...session,- user: { ...session.user, v4BetaEnabled },+ user: {+ ...session.user,+ v4BetaEnabled,+ organizations: session.user!.organizations.map((org) => ({+ ...org,+ plan,+ })),+ },} as Session,headers: {},}),@@ -360,6 +374,33 @@ describe("batched evaluation version selection", () => {);});+ it("limits the selection to the organization plan's data-access window", async () => {+ const context = prepare({ plan: "cloud:hobby" });++ await context.runEvaluation.create({+ projectId,+ query,+ evaluatorIds: [evaluatorId],+ sourceTable: BatchEvalSourceTable.EVENTS,+ });++ const accessFloorFilter = expect.objectContaining({+ type: "datetime",+ column: "startTime",+ operator: ">=",+ value: expect.any(Date),+ });+ expect(context.batchActionCreate).toHaveBeenCalledWith(+ expect.objectContaining({+ data: expect.objectContaining({+ query: expect.objectContaining({+ filter: [rawCommentFilter, accessFloorFilter],+ }),+ }),+ }),+ );+ });+it("rejects evaluator v2 for users outside fast preview", async () => {const context = prepare();@@ -576,6 +617,28 @@ describe("batched evaluation version selection", () => {expect(context.batchActionCreate).not.toHaveBeenCalled();});++ it("validates inherited mappings when no override is sent", async () => {+ const context = prepare({+ v4BetaEnabled: true,+ missingPromptVariable: true,+ });++ await expect(+ context.runEvaluation.create({+ projectId,+ query,+ evaluatorIds: [evaluatorId],+ sourceTable: BatchEvalSourceTable.EVENTS,+ evalVersion: "v2",+ }),+ ).rejects.toMatchObject({+ code: "BAD_REQUEST",+ message: expect.stringContaining('Evaluator "Quality"'),+ });++ expect(context.batchActionCreate).not.toHaveBeenCalled();+ });});describe("batch action history", () => {diff --git a/web/src/features/batch-actions/server/batchEvaluationService.ts b/web/src/features/batch-actions/server/batchEvaluationService.tsnew file mode 100644index 0000000000..437c03eb30--- /dev/null+++ b/web/src/features/batch-actions/server/batchEvaluationService.ts@@ -0,0 +1,259 @@+import type { z } from "zod";+import {+ ActionId,+ BatchActionStatus,+ BatchEvalSourceTable,+ BatchTableNames,+ getEvalTargetObjectFromSourceTable,+ InvalidRequestError,+ type BatchActionQuery,+ type Plan,+} from "@langfuse/shared";+import type { BatchAction, PrismaClient } from "@langfuse/shared/src/db";+import {+ BatchActionQueue,+ QueueJobs,+ applyCommentFilters,+ getObservationsCountFromEventsTable,+ logger,+} from "@langfuse/shared/src/server";+import { env } from "@/src/env.mjs";+import { clampToDataAccessDays } from "@/src/features/entitlements/server";+import { batchEligibleEvaluatorWhere } from "@/src/features/evals/v2/server/evaluators/evaluatorRepository";+import type { CreateObservationBatchEvaluationActionSchema } from "../validation";+import { prepareBatchEvalEvaluatorMappings } from "./prepareBatchEvalEvaluatorMappings";++type CreateBatchEvaluationInput = z.output<+ typeof CreateObservationBatchEvaluationActionSchema+>;++function withDataAccessFloor(+ query: BatchActionQuery,+ plan: Plan | null,+): BatchActionQuery {+ const { accessFloor } = clampToDataAccessDays({ plan });+ if (!accessFloor) return query;++ return {+ ...query,+ filter: [+ ...(query.filter ?? []),+ {+ type: "datetime",+ column: "startTime",+ operator: ">=",+ value: accessFloor,+ },+ ],+ };+}++function missingEvaluatorsMessage(params: {+ missingEvaluatorIds: string[];+ isEvaluatorV2: boolean;+ sourceTable: BatchEvalSourceTable;+}): string {+ const { missingEvaluatorIds, isEvaluatorV2, sourceTable } = params;+ const subject =+ missingEvaluatorIds.length > 0+ ? `Evaluators [${missingEvaluatorIds.join(", ")}]`+ : "Selected evaluators";++ if (isEvaluatorV2) {+ return `${subject} are missing or incompatible with batch evaluation.`;+ }++ const scopeLabel =+ sourceTable === BatchEvalSourceTable.EVENTS ? "observation" : "experiment";+ return `${subject} are missing or not ${scopeLabel}-scoped.`;+}++/**+ * Queues a batch evaluation over the observations matching a table query.+ * Callers authorize the actor (project RBAC, rollout gates) before calling.+ */+export class BatchEvaluationService {+ constructor(+ private readonly prisma: PrismaClient,+ private readonly audit: (batchAction: BatchAction) => Promise<void>,+ ) {}++ async create(+ input: CreateBatchEvaluationInput,+ /** `plan` limits the selection to the organization's data-access window. */+ actor: { userId: string; plan: Plan | null },+ ): Promise<{ id: string }> {+ const {+ projectId,+ sourceTable,+ evaluatorMappings: rawEvaluatorMappings,+ sampling,+ rowLimit,+ } = input;++ if (env.LANGFUSE_MIGRATION_V4_ALLOW_PREVIEW_OPT_IN !== "true") {+ throw new InvalidRequestError(+ "Events table is not enabled for this instance.",+ );+ }++ const query = withDataAccessFloor(input.query, actor.plan);+ const evaluatorIds = await this.findEvaluatorIds(input, sourceTable);++ // Every v2 evaluator is validated, including those that inherit their+ // version mapping, so a run never executes with unmapped prompt variables.+ const overrides = new Map(+ (rawEvaluatorMappings ?? []).map((m) => [m.evaluatorId, m]),+ );+ const evaluatorMappings =+ input.evalVersion === "v2"+ ? await prepareBatchEvalEvaluatorMappings({+ prisma: this.prisma,+ projectId,+ mappings: evaluatorIds.map(+ (evaluatorId) =>+ overrides.get(evaluatorId) ?? {+ evaluatorId,+ variableMapping: null,+ },+ ),+ })+ : undefined;++ const observationCount = await this.countMatchingObservations({+ projectId,+ query,+ sourceTable,+ });+ if (+ rowLimit === undefined &&+ observationCount > env.LANGFUSE_MAX_HISTORIC_EVAL_CREATION_LIMIT+ ) {+ throw new InvalidRequestError(+ `Too many observations selected. Maximum allowed is ${env.LANGFUSE_MAX_HISTORIC_EVAL_CREATION_LIMIT}, but ${observationCount} observations match your filters. Please refine your filters to reduce the count.`,+ );+ }++ const batchConfig = {+ evaluatorIds,+ ...(input.evalVersion ? { evalVersion: input.evalVersion } : {}),+ ...(evaluatorMappings ? { evaluatorMappings } : {}),+ ...(sampling !== undefined ? { sampling } : {}),+ ...(rowLimit !== undefined ? { rowLimit } : {}),+ };++ logger.info("Creating observation-run-batched-evaluation action", {+ projectId,+ evaluatorCount: evaluatorIds.length,+ evaluatorIds,+ });++ const batchAction = await this.prisma.batchAction.create({+ data: {+ projectId,+ userId: actor.userId,+ actionType: ActionId.ObservationBatchEvaluation,+ tableName: BatchTableNames.Events,+ status: BatchActionStatus.Queued,+ query,+ config: batchConfig,+ },+ });++ await this.audit(batchAction);++ await BatchActionQueue.getInstance()?.add(+ QueueJobs.BatchActionProcessingJob,+ {+ id: batchAction.id,+ name: QueueJobs.BatchActionProcessingJob,+ timestamp: new Date(),+ payload: {+ actionId: ActionId.ObservationBatchEvaluation,+ batchActionId: batchAction.id,+ projectId,+ cutoffCreatedAt: new Date(),+ query,+ ...batchConfig,+ },+ },+ {+ jobId: batchAction.id,+ },+ );++ return { id: batchAction.id };+ }++ private async findEvaluatorIds(+ input: CreateBatchEvaluationInput,+ sourceTable: BatchEvalSourceTable,+ ): Promise<string[]> {+ const { projectId } = input;+ const requestedEvaluatorIds = Array.from(new Set(input.evaluatorIds));++ const found =+ input.evalVersion === "v2"+ ? await this.prisma.evaluator.findMany({+ where: {+ id: { in: requestedEvaluatorIds },+ projectId,+ ...batchEligibleEvaluatorWhere,+ },+ select: { id: true },+ })+ : await this.prisma.evaluationRule.findMany({+ where: {+ id: { in: requestedEvaluatorIds },+ projectId,+ targetObject: getEvalTargetObjectFromSourceTable(sourceTable),+ },+ select: { id: true },+ });+ const evaluatorIds = found.map((e) => e.id);++ if (evaluatorIds.length !== requestedEvaluatorIds.length) {+ const foundIds = new Set(evaluatorIds);+ throw new InvalidRequestError(+ missingEvaluatorsMessage({+ missingEvaluatorIds: requestedEvaluatorIds.filter(+ (id) => !foundIds.has(id),+ ),+ isEvaluatorV2: input.evalVersion === "v2",+ sourceTable,+ }),+ );+ }++ return evaluatorIds;+ }++ private async countMatchingObservations(params: {+ projectId: string;+ query: BatchActionQuery;+ sourceTable: BatchEvalSourceTable;+ }): Promise<number> {+ const { projectId, query, sourceTable } = params;++ // Event comments live in Postgres, so resolve them for the preflight+ // count while retaining the original query for the queued worker.+ const commentFilterResult =+ sourceTable === BatchEvalSourceTable.EVENTS+ ? await applyCommentFilters({+ filterState: query.filter ?? [],+ prisma: this.prisma,+ projectId,+ objectType: "OBSERVATION",+ })+ : null;++ if (commentFilterResult?.hasNoMatches) return 0;++ return getObservationsCountFromEventsTable({+ projectId,+ filter: commentFilterResult?.filterState ?? query.filter ?? [],+ searchQuery: query.searchQuery,+ searchType: query.searchType,+ });+ }+}diff --git a/web/src/features/batch-actions/server/runEvaluationRouter.ts b/web/src/features/batch-actions/server/runEvaluationRouter.tsindex a2673de079..cad421f9a5 100644--- a/web/src/features/batch-actions/server/runEvaluationRouter.ts+++ b/web/src/features/batch-actions/server/runEvaluationRouter.ts@@ -1,30 +1,14 @@-/* eslint-disable no-nested-ternary */import { auditLog } from "@/src/features/audit-logs/server";import { throwIfNoProjectAccess } from "@/src/features/rbac";import {createTRPCRouter,protectedProjectProcedure,} from "@/src/server/api/trpc";-import {- BatchActionQueue,- logger,- QueueJobs,- applyCommentFilters,- getObservationsCountFromEventsTable,-} from "@langfuse/shared/src/server";+import { logger } from "@langfuse/shared/src/server";import { TRPCError } from "@trpc/server";-import {- BatchTableNames,- BatchActionStatus,- ActionId,- BatchEvalSourceTable,- getEvalTargetObjectFromSourceTable,- InvalidRequestError,-} from "@langfuse/shared";-import { env } from "@/src/env.mjs";+import { ActionId, InvalidRequestError } from "@langfuse/shared";import { CreateObservationBatchEvaluationActionSchema } from "../validation";-import { batchEligibleEvaluatorWhere } from "@/src/features/evals/v2/server/evaluators/evaluatorRepository";-import { prepareBatchEvalEvaluatorMappings } from "./prepareBatchEvalEvaluatorMappings";+import { BatchEvaluationService } from "./batchEvaluationService";export const runEvaluationRouter = createTRPCRouter({create: protectedProjectProcedure@@ -37,32 +21,6 @@ export const runEvaluationRouter = createTRPCRouter({scope: "evaluationRule:CUD",});- const {- projectId,- query,- evaluatorIds: rawEvaluatorIds,- sourceTable = BatchEvalSourceTable.EVENTS,- evaluatorMappings: rawEvaluatorMappings,- sampling,- rowLimit,- } = input;-- if (env.LANGFUSE_MIGRATION_V4_ALLOW_PREVIEW_OPT_IN !== "true") {- throw new TRPCError({- code: "BAD_REQUEST",- message: "Events table is not enabled for this instance.",- });- }-- // Derive targetObject from sourceTable- const targetObject = getEvalTargetObjectFromSourceTable(sourceTable);- const scopeLabel =- sourceTable === BatchEvalSourceTable.EVENTS- ? "observation"- : "experiment";-- const requestedEvaluatorIds = Array.from(new Set(rawEvaluatorIds));-if (input.evalVersion === "v2" &&ctx.session.user.v4BetaEnabled !== true@@ -73,157 +31,27 @@ export const runEvaluationRouter = createTRPCRouter({});}- const evaluatorIds = (- input.evalVersion === "v2"- ? await ctx.prisma.evaluator.findMany({- where: {- id: { in: requestedEvaluatorIds },- projectId,- ...batchEligibleEvaluatorWhere,- },- select: { id: true },- })- : await ctx.prisma.evaluationRule.findMany({- where: {- id: {- in: requestedEvaluatorIds,- },- projectId,- targetObject,- },- select: {- id: true,- },- })- ).map((e) => e.id);-- if (evaluatorIds.length !== requestedEvaluatorIds.length) {- const foundIds = new Set(evaluatorIds);- const missingEvaluatorIds = requestedEvaluatorIds.filter(- (id) => !foundIds.has(id),- );-- throw new TRPCError({- code: "BAD_REQUEST",- message:- missingEvaluatorIds.length > 0- ? input.evalVersion === "v2"- ? `Evaluators [${missingEvaluatorIds.join(", ")}] are missing or incompatible with batch evaluation.`- : `Evaluators [${missingEvaluatorIds.join(", ")}] are missing or not ${scopeLabel}-scoped.`- : input.evalVersion === "v2"- ? "Selected evaluators are missing or incompatible with batch evaluation."- : `Selected evaluators are missing or not ${scopeLabel}-scoped.`,- });- }-- const evaluatorMappings =- input.evalVersion === "v2" && rawEvaluatorMappings- ? await prepareBatchEvalEvaluatorMappings({- prisma: ctx.prisma,- projectId,- mappings: rawEvaluatorMappings,- })- : undefined;-- // Event comments live in Postgres, so resolve them for the preflight- // count while retaining the original query for the queued worker.- const commentFilterResult =- sourceTable === BatchEvalSourceTable.EVENTS- ? await applyCommentFilters({- filterState: query.filter ?? [],- prisma: ctx.prisma,- projectId,- objectType: "OBSERVATION",- })- : null;-- const countQueryOpts = {- projectId,- filter: commentFilterResult?.filterState ?? query.filter ?? [],- searchQuery: query.searchQuery,- searchType: query.searchType,- };-- const observationCount = commentFilterResult?.hasNoMatches- ? 0- : await getObservationsCountFromEventsTable(countQueryOpts);-- if (- rowLimit === undefined &&- observationCount > env.LANGFUSE_MAX_HISTORIC_EVAL_CREATION_LIMIT- ) {- throw new TRPCError({- code: "BAD_REQUEST",- message: `Too many observations selected. Maximum allowed is ${env.LANGFUSE_MAX_HISTORIC_EVAL_CREATION_LIMIT}, but ${observationCount} observations match your filters. Please refine your filters to reduce the count.`,- });- }-- const userId = ctx.session.user.id;- const batchConfig = {- evaluatorIds,- ...(input.evalVersion ? { evalVersion: input.evalVersion } : {}),- ...(evaluatorMappings ? { evaluatorMappings } : {}),- ...(sampling !== undefined ? { sampling } : {}),- ...(rowLimit !== undefined ? { rowLimit } : {}),- };-- logger.info(- "[TRPC] Creating observation-run-batched-evaluation action",- {- projectId,- evaluatorCount: evaluatorIds.length,- evaluatorIds,- },+ const service = new BatchEvaluationService(ctx.prisma, (batchAction) =>+ auditLog({+ session: ctx.session,+ resourceType: "batchAction",+ resourceId: batchAction.id,+ projectId: batchAction.projectId,+ action: ActionId.ObservationBatchEvaluation,+ after: batchAction,+ }),);- const batchAction = await ctx.prisma.batchAction.create({- data: {- projectId,- userId,- actionType: ActionId.ObservationBatchEvaluation,- tableName: BatchTableNames.Events,- status: BatchActionStatus.Queued,- query,- config: batchConfig,- },- });+ const plan = ctx.session.user.admin+ ? null+ : (ctx.session.user.organizations.find(+ (org) => org.id === ctx.session.orgId,+ )?.plan ?? null);- await auditLog({- session: ctx.session,- resourceType: "batchAction",- resourceId: batchAction.id,- projectId,- action: ActionId.ObservationBatchEvaluation,- after: batchAction,+ return await service.create(input, {+ userId: ctx.session.user.id,+ plan,});-- await BatchActionQueue.getInstance()?.add(- QueueJobs.BatchActionProcessingJob,- {- id: batchAction.id,- name: QueueJobs.BatchActionProcessingJob,- timestamp: new Date(),- payload: {- actionId: ActionId.ObservationBatchEvaluation,- batchActionId: batchAction.id,- projectId,- cutoffCreatedAt: new Date(),- query,- evaluatorIds: batchConfig.evaluatorIds,- ...(batchConfig.evalVersion- ? { evalVersion: batchConfig.evalVersion }- : {}),- ...(evaluatorMappings ? { evaluatorMappings } : {}),- ...(sampling !== undefined ? { sampling } : {}),- ...(rowLimit !== undefined ? { rowLimit } : {}),- },- },- {- jobId: batchAction.id,- },- );-- return { id: batchAction.id };} catch (e) {logger.error(e);if (e instanceof TRPCError) {diff --git a/web/src/features/in-app-agent/components/utils/side-effects.ts b/web/src/features/in-app-agent/components/utils/side-effects.tsindex bcd61308c1..7effadb085 100644--- a/web/src/features/in-app-agent/components/utils/side-effects.ts+++ b/web/src/features/in-app-agent/components/utils/side-effects.ts@@ -84,6 +84,7 @@ const IN_APP_AGENT_TOOL_TRPC_INVALIDATION_TARGETS = {langfuse_attachEvaluatorToEvaluationRule: ["evals"],langfuse_detachEvaluatorFromEvaluationRule: ["evals"],langfuse_deleteEvaluationRule: ["evals"],+ langfuse_runBatchEvaluation: ["evals"],langfuse_listExperiments: [],langfuse_listExperimentItems: [],langfuse_submitFeedback: ["scores", "scoreAnalytics"],diff --git a/web/src/features/in-app-agent/components/utils/utils.ts b/web/src/features/in-app-agent/components/utils/utils.tsindex e748b30183..2afbcbe8d5 100644--- a/web/src/features/in-app-agent/components/utils/utils.ts+++ b/web/src/features/in-app-agent/components/utils/utils.ts@@ -87,6 +87,7 @@ const IN_APP_AGENT_TOOL_PROGRESS_LABEL_OVERRIDES: Record<string, string> = {proposeRedirect: "Opening page",queryMetrics: "Checking metrics",read: "Reading file",+ runBatchEvaluation: "Running batch evaluation",submitFeedback: "Submitting user feedback",testEvaluator: "Testing evaluator",testExternalMediaStorage: "Testing external media storage",diff --git a/web/src/features/mcp/server/bootstrap.ts b/web/src/features/mcp/server/bootstrap.tsindex 42dd3f91c1..ff31a53e4e 100644--- a/web/src/features/mcp/server/bootstrap.ts+++ b/web/src/features/mcp/server/bootstrap.ts@@ -23,7 +23,7 @@ import { metricsFeature } from "./metrics";import { modelsFeature } from "./models";import { llmConnectionsFeature } from "./llmConnections";import { mediaFeature } from "./media";-import { evalsFeature } from "./evals";+import { batchEvaluationsFeature, evalsFeature } from "./evals";import { dashboardWidgetsFeature } from "./dashboardWidgets";import { feedbackFeature } from "./feedback";import { experimentsFeature } from "./experiments";@@ -45,6 +45,7 @@ const MCP_FEATURES = [llmConnectionsFeature,mediaFeature,evalsFeature,+ batchEvaluationsFeature,dashboardWidgetsFeature,feedbackFeature,experimentsFeature,diff --git a/web/src/features/mcp/server/evals/batch-evaluation-service.ts b/web/src/features/mcp/server/evals/batch-evaluation-service.tsnew file mode 100644index 0000000000..5b660bcc40--- /dev/null+++ b/web/src/features/mcp/server/evals/batch-evaluation-service.ts@@ -0,0 +1,19 @@+import { auditLog } from "@/src/features/audit-logs/server";+import { BatchEvaluationService } from "@/src/features/batch-actions/server/batchEvaluationService";+import { ActionId } from "@langfuse/shared";+import { prisma } from "@langfuse/shared/src/db";+import type { ServerContext } from "../../types";++export function createMcpBatchEvaluationService(context: ServerContext) {+ return new BatchEvaluationService(prisma, (batchAction) =>+ auditLog({+ action: ActionId.ObservationBatchEvaluation,+ resourceType: "batchAction",+ resourceId: batchAction.id,+ projectId: context.projectId,+ orgId: context.orgId,+ apiKeyId: context.apiKeyId,+ after: batchAction,+ }),+ );+}diff --git a/web/src/features/mcp/server/evals/index.ts b/web/src/features/mcp/server/evals/index.tsindex dda0457ae4..a46d65db58 100644--- a/web/src/features/mcp/server/evals/index.ts+++ b/web/src/features/mcp/server/evals/index.ts@@ -1,3 +1,4 @@+import { env } from "@/src/env.mjs";import type { McpFeatureModule } from "../registry";import {listEvaluatorsTool,@@ -47,6 +48,10 @@ import {handleDetachEvaluatorFromEvaluationRule,} from "./tools/manageEvaluationRuleEvaluators";import { handleTestEvaluator, testEvaluatorTool } from "./tools/testEvaluator";+import {+ handleRunBatchEvaluation,+ runBatchEvaluationTool,+} from "./tools/runBatchEvaluation";export const evalsFeature = {name: "evals",@@ -105,3 +110,17 @@ export const evalsFeature = {},],} as const satisfies McpFeatureModule;++export const batchEvaluationsFeature = {+ name: "batchEvaluations",+ description:+ "Run evaluators over existing observations in the current Langfuse project",+ tools: [+ {+ definition: runBatchEvaluationTool,+ handler: handleRunBatchEvaluation,+ },+ ],+ isEnabled: async () =>+ env.LANGFUSE_MIGRATION_V4_ALLOW_PREVIEW_OPT_IN === "true",+} as const satisfies McpFeatureModule;diff --git a/web/src/features/mcp/server/evals/tools/runBatchEvaluation.ts b/web/src/features/mcp/server/evals/tools/runBatchEvaluation.tsnew file mode 100644index 0000000000..f3233bdd6e--- /dev/null+++ b/web/src/features/mcp/server/evals/tools/runBatchEvaluation.ts@@ -0,0 +1,84 @@+import { z } from "zod";+import { CreateObservationBatchEvaluationActionSchema } from "@/src/features/batch-actions/validation";+import { defineTool } from "../../../core/define-tool";+import { runMcpTool } from "../../../core/run-mcp-tool";+import {+ buildStartTimeFilters,+ ObservationMcpBatchQueryFilterSchema,+ ObservationMcpFilterBaseSchema,+ OptionalStartTimeRangeShape,+} from "../../observations/filter-schema";+import { createMcpBatchEvaluationService } from "../batch-evaluation-service";++const { evaluatorIds, sampling, rowLimit } =+ CreateObservationBatchEvaluationActionSchema.shape;++const FILTER_DESCRIPTION =+ "Observation filters in the listObservations filter format; call getObservationFilterSchema for accepted columns and operators. The full-text `matches` operator is not supported here. Bound the selection by time with fromStartTime and toStartTime.";++const RunBatchEvaluationBaseSchema = z+ .object({+ evaluatorIds: evaluatorIds.describe(+ "IDs of the evaluators to run, as returned by listEvaluators. Each must be observation-scoped and not attached to a trace- or dataset-level evaluation rule.",+ ),+ ...OptionalStartTimeRangeShape,+ filter: z+ .array(ObservationMcpFilterBaseSchema)+ .optional()+ .describe(FILTER_DESCRIPTION),+ sampling: sampling.describe(+ "Fraction of matching observations to evaluate, between 0 and 1. Defaults to 1.",+ ),+ rowLimit: rowLimit.describe(+ "Maximum number of observations to evaluate. Without it, the run is rejected when more observations match than the instance's batch evaluation limit.",+ ),+ })+ .strict();++const RunBatchEvaluationInputSchema = RunBatchEvaluationBaseSchema.extend({+ filter: z+ .array(ObservationMcpBatchQueryFilterSchema)+ .optional()+ .describe(FILTER_DESCRIPTION),+}).transform(({ fromStartTime, toStartTime, filter, ...input }) => ({+ ...input,+ filter: [+ ...(filter ?? []),+ ...buildStartTimeFilters({ fromStartTime, toStartTime }),+ ],+}));++export const [runBatchEvaluationTool, handleRunBatchEvaluation] = defineTool({+ name: "runBatchEvaluation",+ description: [+ "Run one or more saved evaluators over the existing observations that match a filter in the current Langfuse project, like Run Evaluation in the observations table.",+ "Preview the selection with listObservations using the same filter and time range before running. Observations older than the plan's data-access window are never evaluated.",+ "The run is queued and processed asynchronously; this returns the batch action id. Results are written as scores on the evaluated observations, and each score and evaluator trace carries the batch action id as batch_action_id metadata.",+ "This executes every selected evaluator on every selected observation and may incur model or code execution cost.",+ ].join(" "),+ action: "evaluationRule:CUD",+ baseSchema: RunBatchEvaluationBaseSchema,+ inputSchema: RunBatchEvaluationInputSchema,+ handler: (input, context) =>+ runMcpTool({+ spanName: "mcp.evaluators.run_batch",+ context,+ attributes: {+ "mcp.evaluator_count": input.evaluatorIds.length,+ },+ fn: () =>+ createMcpBatchEvaluationService(context).create(+ {+ projectId: context.projectId,+ query: { filter: input.filter, orderBy: null },+ evaluatorIds: input.evaluatorIds,+ sourceTable: "events",+ evalVersion: "v2",+ sampling: input.sampling,+ rowLimit: input.rowLimit,+ },+ { userId: context.userId ?? context.apiKeyId, plan: context.plan },+ ),+ }),+ expensiveHint: true,+});diff --git a/web/src/features/mcp/server/observations/filter-schema.ts b/web/src/features/mcp/server/observations/filter-schema.tsnew file mode 100644index 0000000000..4985983618--- /dev/null+++ b/web/src/features/mcp/server/observations/filter-schema.ts@@ -0,0 +1,278 @@+import {+ isoDateTime,+ OBSERVATION_MCP_ALLOWED_EVENTS_TABLE_FILTER_COLUMNS,+ booleanFilter,+ coerceLegacyEmptyMetadataFilters,+ eventsTableCols,+ eventsTableSingleFilter,+ eventsTableStringFilter,+ eventsTableStringObjectFilter,+ filterOperators,+ FTS_MATCH_OPERATOR,+ numberFilter,+ singleFilter,+ stringFilter,+ stringObjectFilter,+ stringOptionsFilter,+ timeFilter,+ type ColumnDefinition,+} from "@langfuse/shared";+import { z } from "zod";+import {+ OBSERVATION_MCP_FTS_COLUMNS,+ OBSERVATION_MCP_INDEXED_IO_COLUMNS,+} from "./tools/getObservationFilterSchema";++const addIssues = (ctx: z.RefinementCtx, issues: z.core.$ZodIssue[]) => {+ for (const issue of issues) {+ ctx.addIssue({ code: "custom", path: issue.path, message: issue.message });+ }+};++const OBSERVATION_MCP_FILTER_COLUMN_TYPES = new Map(+ eventsTableCols+ .filter((column) =>+ OBSERVATION_MCP_ALLOWED_EVENTS_TABLE_FILTER_COLUMNS.has(column.id),+ )+ .map((column) => [+ column.id === "traceTags" ? "tags" : column.id,+ column.type,+ ]),+);++const OBSERVATION_MCP_FILTER_EXAMPLE = {+ column: "totalCost",+ operator: ">",+ value: 0.0029,+} satisfies Omit<z.infer<typeof numberFilter>, "type">;++const OBSERVATION_MCP_FILTER_EXAMPLE_WITH_TYPE = {+ type: "number",+ ...OBSERVATION_MCP_FILTER_EXAMPLE,+} satisfies z.infer<typeof numberFilter>;++const OBSERVATION_MCP_FILTER_EXAMPLE_JSON = JSON.stringify(+ OBSERVATION_MCP_FILTER_EXAMPLE,+);++const OBSERVATION_MCP_FILTER_EXAMPLE_WITH_TYPE_JSON = JSON.stringify(+ OBSERVATION_MCP_FILTER_EXAMPLE_WITH_TYPE,+);++const OBSERVATION_MCP_FILTER_SCHEMA_BY_TYPE = {+ datetime: (column: string, requireType = false) =>+ timeFilter.omit({ type: true, column: true }).extend({+ type: requireType+ ? z.literal("datetime")+ : z.literal("datetime").optional(),+ column: z.literal(column),+ }),+ string: (column: string, requireType = false) => {+ const filterSchema = OBSERVATION_MCP_INDEXED_IO_COLUMNS.has(column)+ ? eventsTableStringFilter.extend({+ operator: z.enum(["=", FTS_MATCH_OPERATOR]),+ })+ : stringFilter;++ return filterSchema.omit({ type: true, column: true }).extend({+ type: requireType ? z.literal("string") : z.literal("string").optional(),+ column: z.literal(column),+ });+ },+ stringOptions: (column: string, requireType = false) =>+ stringOptionsFilter.omit({ type: true, column: true }).extend({+ type: requireType+ ? z.literal("stringOptions")+ : z.literal("stringOptions").optional(),+ column: z.literal(column),+ }),+ arrayOptions: (column: string, requireType = false) =>+ z.object({+ operator: z.enum(filterOperators.arrayOptions),+ value: z.array(z.string()),+ type: requireType+ ? z.literal("arrayOptions")+ : z.literal("arrayOptions").optional(),+ column: z.literal(column),+ }),+ number: (column: string, requireType = false) =>+ numberFilter.omit({ type: true, column: true }).extend({+ type: requireType ? z.literal("number") : z.literal("number").optional(),+ column: z.literal(column),+ }),+ stringObject: (column: string, requireType = false) => {+ const filterSchemaBase = OBSERVATION_MCP_FTS_COLUMNS.has(column)+ ? eventsTableStringObjectFilter+ : stringObjectFilter;++ return filterSchemaBase.omit({ type: true, column: true }).extend({+ type: requireType+ ? z.literal("stringObject")+ : z.literal("stringObject").optional(),+ column: z.literal(column),+ });+ },+ boolean: (column: string, requireType = false) =>+ booleanFilter.omit({ type: true, column: true }).extend({+ type: requireType+ ? z.literal("boolean")+ : z.literal("boolean").optional(),+ column: z.literal(column),+ }),+} satisfies Partial<+ Record<+ ColumnDefinition["type"],+ (column: string, requireType?: boolean) => z.ZodType+ >+>;++type ObservationMcpFilterType =+ keyof typeof OBSERVATION_MCP_FILTER_SCHEMA_BY_TYPE;++const isObservationMcpFilterType = (+ type: string,+): type is ObservationMcpFilterType =>+ type in OBSERVATION_MCP_FILTER_SCHEMA_BY_TYPE;++export const ObservationMcpFilterBaseSchema = z+ .object({+ column: z.string(),+ operator: z.string(),+ value: z.any(),+ type: z.string().optional(),+ key: z+ .string()+ .optional()+ .describe(+ "Metadata/object key to filter on. Required when column is metadata.",+ ),+ })+ .describe(+ `Advanced observation filter object. Example: ${OBSERVATION_MCP_FILTER_EXAMPLE_JSON}. The explicit form ${OBSERVATION_MCP_FILTER_EXAMPLE_WITH_TYPE_JSON} is also accepted.`,+ );++const normalizeExactObservationIdFilter = (filter: unknown) => {+ if (!filter || typeof filter !== "object" || Array.isArray(filter)) {+ return filter;+ }++ const candidate = filter as Record<string, unknown>;+ if (+ candidate.column !== "id" ||+ candidate.operator !== "=" ||+ typeof candidate.value !== "string" ||+ (candidate.type !== undefined && candidate.type !== "string")+ ) {+ return filter;+ }++ return {+ ...candidate,+ type: "stringOptions",+ operator: "any of",+ value: [candidate.value],+ };+};++export const ObservationMcpFilterSchema = z.preprocess(+ normalizeExactObservationIdFilter,+ z+ .object({ column: z.string() })+ .loose()+ .superRefine((filter, ctx) => {+ if (!OBSERVATION_MCP_FILTER_COLUMN_TYPES.has(filter.column)) {+ ctx.addIssue({+ code: "custom",+ path: ["column"],+ message: `Invalid observation filter column "${filter.column}". Call getObservationFilterSchema for accepted columns.`,+ });+ return;+ }++ const type = OBSERVATION_MCP_FILTER_COLUMN_TYPES.get(filter.column);++ if (!type || !isObservationMcpFilterType(type)) {+ ctx.addIssue({+ code: "custom",+ path: ["type"],+ message: `Invalid observation filter type "${String(type)}" for column "${filter.column}".`,+ });+ return;+ }++ const filterParseResult = OBSERVATION_MCP_FILTER_SCHEMA_BY_TYPE[type](+ filter.column,+ Boolean(filter.type),+ ).safeParse(filter);++ if (!filterParseResult.success) {+ addIssues(ctx, filterParseResult.error.issues);+ }+ })+ .transform((filter) => {+ const type =+ filter.type ?? OBSERVATION_MCP_FILTER_COLUMN_TYPES.get(filter.column);++ const reshaped =+ filter.column === "tags"+ ? { ...filter, type, column: "traceTags" }+ : { ...filter, type };+ // Legacy `contains ""` presence spelling → `is set` before validation.+ const [coerced] = coerceLegacyEmptyMetadataFilters([+ reshaped,+ ]) as unknown[];++ return eventsTableSingleFilter.parse(coerced);+ }),+);++/**+ * Observation filter restricted to the generic filter contract that batch+ * action queries carry, which excludes the events-only full-text operator.+ */+export const ObservationMcpBatchQueryFilterSchema =+ ObservationMcpFilterSchema.transform((filter, ctx) => {+ const parsed = singleFilter.safeParse(filter);+ if (parsed.success) return parsed.data;+ addIssues(ctx, parsed.error.issues);+ return z.NEVER;+ });++export const OptionalStartTimeRangeShape = {+ fromStartTime: isoDateTime({ offset: true })+ .optional()+ .describe(+ "Optional inclusive lower bound on observation startTime, as an ISO 8601 timestamp with timezone (Z or an explicit offset). Use the same time window as the observations you intend to search.",+ ),+ toStartTime: isoDateTime({ offset: true })+ .optional()+ .describe(+ "Optional exclusive upper bound on observation startTime, as an ISO 8601 timestamp with timezone. For a full calendar day, use midnight at the start of the next day, not 23:59:59.",+ ),+};++export const buildStartTimeFilters = (params: {+ fromStartTime?: string | Date;+ toStartTime?: string | Date;+}): z.infer<typeof timeFilter>[] => [+ ...(params.fromStartTime+ ? [+ {+ column: "startTime",+ operator: ">=" as const,+ value: new Date(params.fromStartTime),+ type: "datetime" as const,+ },+ ]+ : []),+ ...(params.toStartTime+ ? [+ {+ column: "startTime",+ operator: "<" as const,+ value: new Date(params.toStartTime),+ type: "datetime" as const,+ },+ ]+ : []),+];diff --git a/web/src/features/mcp/server/observations/tools/getObservationFilterValues.ts b/web/src/features/mcp/server/observations/tools/getObservationFilterValues.tsindex 5fd13ee3d4..b08b8e7803 100644--- a/web/src/features/mcp/server/observations/tools/getObservationFilterValues.ts+++ b/web/src/features/mcp/server/observations/tools/getObservationFilterValues.ts@@ -1,9 +1,7 @@import {- isoDateTime,InvalidRequestError,ObservationTypeDomain,isNumericEventsTableColumnId,- type timeFilter,} from "@langfuse/shared";import { z } from "zod";import {@@ -18,6 +16,10 @@ import {type ObservationMcpFilterColumn,} from "../schema";import { clampToDataAccessDays } from "@/src/features/entitlements/server";+import {+ buildStartTimeFilters,+ OptionalStartTimeRangeShape,+} from "../filter-schema";const OBSERVATION_MCP_FILTER_VALUE_COLUMNS = ["name",@@ -47,16 +49,7 @@ type FilterValueColumn = z.infer<typeof FilterValueColumnSchema>;const GetObservationFilterValuesBaseSchema = z.object({column: FilterValueColumnSchema,- fromStartTime: isoDateTime({ offset: true })- .optional()- .describe(- "Optional inclusive lower bound on observation startTime, as an ISO 8601 timestamp with timezone (Z or an explicit offset). Use the same time window as the observations you intend to search.",- ),- toStartTime: isoDateTime({ offset: true })- .optional()- .describe(- "Optional exclusive upper bound on observation startTime, as an ISO 8601 timestamp with timezone. For a full calendar day, use midnight at the start of the next day, not 23:59:59.",- ),+ ...OptionalStartTimeRangeShape,observationType: ObservationTypeDomain.optional(),isRootObservation: z.boolean().optional(),hasParentObservation: z.boolean().optional(),@@ -69,33 +62,6 @@ type FilterOption = {count?: number;};-const buildStartTimeFilter = (params: {- fromStartTime?: string;- toStartTime?: string;-}): z.infer<typeof timeFilter>[] => {- const filters: z.infer<typeof timeFilter>[] = [];-- if (params.fromStartTime) {- filters.push({- column: "startTime",- operator: ">=",- value: new Date(params.fromStartTime),- type: "datetime",- });- }-- if (params.toStartTime) {- filters.push({- column: "startTime",- operator: "<",- value: new Date(params.toStartTime),- type: "datetime",- });- }-- return filters;-};-const normalizeFilterOptions = (values: unknown[],column: FilterValueColumn,@@ -190,7 +156,7 @@ export const [plan: context.plan,fromTimestamp: input.fromStartTime,});- const startTimeFilter = buildStartTimeFilter({+ const startTimeFilter = buildStartTimeFilters({fromStartTime: dataAccessWindow.effectiveFromTimestamp?.toISOString(),toStartTime: input.toStartTime,});diff --git a/web/src/features/mcp/server/observations/tools/listObservations.ts b/web/src/features/mcp/server/observations/tools/listObservations.tsindex 646f7303fc..410c9bed59 100644--- a/web/src/features/mcp/server/observations/tools/listObservations.ts+++ b/web/src/features/mcp/server/observations/tools/listObservations.ts@@ -1,23 +1,8 @@import {isoDateTime,InvalidRequestError,- OBSERVATION_MCP_ALLOWED_EVENTS_TABLE_FILTER_COLUMNS,- booleanFilter,- eventsTableSingleFilter,- eventsTableStringFilter,- eventsTableStringObjectFilter,- eventsTableCols,- coerceLegacyEmptyMetadataFilters,- filterOperators,- FTS_MATCH_OPERATOR,- numberFilter,ObservationLevelDomain,ObservationTypeDomain,- stringFilter,- stringObjectFilter,- stringOptionsFilter,- timeFilter,- type ColumnDefinition,} from "@langfuse/shared";import {getObservationsV2FromEventsTableForPublicApi,@@ -44,9 +29,10 @@ import {projectObservation,} from "../schema";import {- OBSERVATION_MCP_FTS_COLUMNS,- OBSERVATION_MCP_INDEXED_IO_COLUMNS,-} from "./getObservationFilterSchema";+ buildStartTimeFilters,+ ObservationMcpFilterBaseSchema,+ ObservationMcpFilterSchema,+} from "../filter-schema";const ObservationCursorSchema =EncodedObservationsCursorV2String.optional().describe(@@ -56,208 +42,8 @@ const ObservationCursorSchema =const OBSERVATION_MAX_RANGE_MS = 14 * 24 * 60 * 60 * 1000; // 14 daysconst EXPENSIVE_OBSERVATION_MAX_IO_LIMIT = 50;-const OBSERVATION_MCP_FILTER_COLUMN_TYPES = new Map(- eventsTableCols- .filter((column) =>- OBSERVATION_MCP_ALLOWED_EVENTS_TABLE_FILTER_COLUMNS.has(column.id),- )- .map((column) => [- column.id === "traceTags" ? "tags" : column.id,- column.type,- ]),-);--const OBSERVATION_MCP_FILTER_EXAMPLE = {- column: "totalCost",- operator: ">",- value: 0.0029,-} satisfies Omit<z.infer<typeof numberFilter>, "type">;--const OBSERVATION_MCP_FILTER_EXAMPLE_WITH_TYPE = {- type: "number",- ...OBSERVATION_MCP_FILTER_EXAMPLE,-} satisfies z.infer<typeof numberFilter>;--const OBSERVATION_MCP_FILTER_EXAMPLE_JSON = JSON.stringify(- OBSERVATION_MCP_FILTER_EXAMPLE,-);--const OBSERVATION_MCP_FILTER_EXAMPLE_WITH_TYPE_JSON = JSON.stringify(- OBSERVATION_MCP_FILTER_EXAMPLE_WITH_TYPE,-);--const OBSERVATION_MCP_FILTER_SCHEMA_BY_TYPE = {- datetime: (column: string, requireType = false) =>- timeFilter.omit({ type: true, column: true }).extend({- type: requireType- ? z.literal("datetime")- : z.literal("datetime").optional(),- column: z.literal(column),- }),- string: (column: string, requireType = false) => {- const filterSchema = OBSERVATION_MCP_INDEXED_IO_COLUMNS.has(column)- ? eventsTableStringFilter.extend({- operator: z.enum(["=", FTS_MATCH_OPERATOR]),- })- : stringFilter;-- return filterSchema.omit({ type: true, column: true }).extend({- type: requireType ? z.literal("string") : z.literal("string").optional(),- column: z.literal(column),- });- },- stringOptions: (column: string, requireType = false) =>- stringOptionsFilter.omit({ type: true, column: true }).extend({- type: requireType- ? z.literal("stringOptions")- : z.literal("stringOptions").optional(),- column: z.literal(column),- }),- arrayOptions: (column: string, requireType = false) =>- z.object({- operator: z.enum(filterOperators.arrayOptions),- value: z.array(z.string()),- type: requireType- ? z.literal("arrayOptions")- : z.literal("arrayOptions").optional(),- column: z.literal(column),- }),- number: (column: string, requireType = false) =>- numberFilter.omit({ type: true, column: true }).extend({- type: requireType ? z.literal("number") : z.literal("number").optional(),- column: z.literal(column),- }),- stringObject: (column: string, requireType = false) => {- const filterSchemaBase = OBSERVATION_MCP_FTS_COLUMNS.has(column)- ? eventsTableStringObjectFilter- : stringObjectFilter;-- return filterSchemaBase.omit({ type: true, column: true }).extend({- type: requireType- ? z.literal("stringObject")- : z.literal("stringObject").optional(),- column: z.literal(column),- });- },- boolean: (column: string, requireType = false) =>- booleanFilter.omit({ type: true, column: true }).extend({- type: requireType- ? z.literal("boolean")- : z.literal("boolean").optional(),- column: z.literal(column),- }),-} satisfies Partial<- Record<- ColumnDefinition["type"],- (column: string, requireType?: boolean) => z.ZodType- >->;--type ObservationMcpFilterType =- keyof typeof OBSERVATION_MCP_FILTER_SCHEMA_BY_TYPE;--const isObservationMcpFilterType = (- type: string,-): type is ObservationMcpFilterType =>- type in OBSERVATION_MCP_FILTER_SCHEMA_BY_TYPE;--const ObservationMcpFilterBaseSchema = z- .object({- column: z.string(),- operator: z.string(),- value: z.any(),- type: z.string().optional(),- key: z- .string()- .optional()- .describe(- "Metadata/object key to filter on. Required when column is metadata.",- ),- })- .describe(- `Advanced observation filter object. Example: ${OBSERVATION_MCP_FILTER_EXAMPLE_JSON}. The explicit form ${OBSERVATION_MCP_FILTER_EXAMPLE_WITH_TYPE_JSON} is also accepted.`,- );--const normalizeExactObservationIdFilter = (filter: unknown) => {- if (!filter || typeof filter !== "object" || Array.isArray(filter)) {- return filter;- }-- const candidate = filter as Record<string, unknown>;- if (- candidate.column !== "id" ||- candidate.operator !== "=" ||- typeof candidate.value !== "string" ||- (candidate.type !== undefined && candidate.type !== "string")- ) {- return filter;- }-- return {- ...candidate,- type: "stringOptions",- operator: "any of",- value: [candidate.value],- };-};--const ObservationMcpFilterSchema = z.preprocess(- normalizeExactObservationIdFilter,- z- .object({ column: z.string() })- .loose()- .superRefine((filter, ctx) => {- if (!OBSERVATION_MCP_FILTER_COLUMN_TYPES.has(filter.column)) {- ctx.addIssue({- code: "custom",- path: ["column"],- message: `Invalid observation filter column "${filter.column}". Call getObservationFilterSchema for accepted columns.`,- });- return;- }-- const type = OBSERVATION_MCP_FILTER_COLUMN_TYPES.get(filter.column);-- if (!type || !isObservationMcpFilterType(type)) {- ctx.addIssue({- code: "custom",- path: ["type"],- message: `Invalid observation filter type "${String(type)}" for column "${filter.column}".`,- });- return;- }-- const filterParseResult = OBSERVATION_MCP_FILTER_SCHEMA_BY_TYPE[type](- filter.column,- Boolean(filter.type),- ).safeParse(filter);-- if (!filterParseResult.success) {- for (const issue of filterParseResult.error.issues) {- ctx.addIssue({- code: "custom",- path: issue.path,- message: issue.message,- });- }- }- })- .transform((filter) => {- const type =- filter.type ?? OBSERVATION_MCP_FILTER_COLUMN_TYPES.get(filter.column);-- const reshaped =- filter.column === "tags"- ? { ...filter, type, column: "traceTags" }- : { ...filter, type };- // Legacy `contains ""` presence spelling → `is set` before validation.- const [coerced] = coerceLegacyEmptyMetadataFilters([- reshaped,- ]) as unknown[];-- return eventsTableSingleFilter.parse(coerced);- }),-);+const OBSERVATION_FILTER_DESCRIPTION =+ "Advanced filters. Each item must be an object with column, operator, value, optional type, and optional key. Type is inferred from getObservationFilterSchema columns when omitted. Use key for metadata filters.";const ListObservationsBaseSchema = z.object({fields: ObservationFieldsSchema,@@ -298,18 +84,14 @@ const ListObservationsBaseSchema = z.object({filter: z.array(ObservationMcpFilterBaseSchema).optional()- .describe(- "Advanced filters. Each item must be an object with column, operator, value, optional type, and optional key. Type is inferred from getObservationFilterSchema columns when omitted. Use key for metadata filters.",- ),+ .describe(OBSERVATION_FILTER_DESCRIPTION),});const ListObservationsInputSchema = ListObservationsBaseSchema.extend({filter: z.array(ObservationMcpFilterSchema).optional()- .describe(- "Advanced filters. Each item must be an object with column, operator, value, optional type, and optional key. Type is inferred from getObservationFilterSchema columns when omitted. Use key for metadata filters.",- ),+ .describe(OBSERVATION_FILTER_DESCRIPTION),});type ListObservationsInput = z.infer<typeof ListObservationsInputSchema>;@@ -370,22 +152,10 @@ export const [listObservationsTool, handleListObservations] = defineTool({const advancedFilters = dataAccessWindow.accessFloor? [...(input.filter ?? []),- {- column: "startTime",- operator: ">=" as const,- value: dataAccessWindow.effectiveFromTimestamp!,- type: "datetime" as const,- },- ...(input.toStartTime- ? [- {- column: "startTime",- operator: "<" as const,- value: new Date(input.toStartTime),- type: "datetime" as const,- },- ]- : []),+ ...buildStartTimeFilters({+ fromStartTime: dataAccessWindow.effectiveFromTimestamp,+ toStartTime: input.toStartTime,+ }),]: input.filter;diff --git a/worker/src/features/batchAction/processBatchedObservationEval.ts b/worker/src/features/batchAction/processBatchedObservationEval.tsindex 91cd3e9928..e42865be64 100644--- a/worker/src/features/batchAction/processBatchedObservationEval.ts+++ b/worker/src/features/batchAction/processBatchedObservationEval.ts@@ -64,6 +64,7 @@ export async function processBatchedObservationEval(params: {schedulerDeps,executionMode: "MANUAL",executionScopeId: batchActionId,+ batchActionId,});}),),diff --git a/worker/src/features/evaluation/observationEval/__tests__/createSchedulerDeps.test.ts b/worker/src/features/evaluation/observationEval/__tests__/createSchedulerDeps.test.tsindex 1dc0b4cbf8..adf880de57 100644--- a/worker/src/features/evaluation/observationEval/__tests__/createSchedulerDeps.test.ts+++ b/worker/src/features/evaluation/observationEval/__tests__/createSchedulerDeps.test.ts@@ -79,7 +79,7 @@ describe("createObservationEvalSchedulerDeps", () => {expect(getLLMQueueInstance).not.toHaveBeenCalled();});- it("includes a mapping override on the observation eval payload", async () => {+ it("includes batch-run fields on the observation eval payload", async () => {const { createObservationEvalSchedulerDeps } =await import("../createSchedulerDeps");const variableMapping = [@@ -94,6 +94,7 @@ describe("createObservationEvalSchedulerDeps", () => {evalTemplateType: EvalTemplateType.LLM_AS_JUDGE,evaluatorId: "evaluator-1",variableMapping,+ batchActionId: "batch-action-1",});expect(addToLLMQueue).toHaveBeenCalledWith(@@ -102,6 +103,7 @@ describe("createObservationEvalSchedulerDeps", () => {payload: expect.objectContaining({evaluatorId: "evaluator-1",variableMapping,+ batchActionId: "batch-action-1",}),}),{ delay: 0 },diff --git a/worker/src/features/evaluation/observationEval/__tests__/observationEval.e2e.test.ts b/worker/src/features/evaluation/observationEval/__tests__/observationEval.e2e.test.tsindex 0eedf1e10e..18c5231ae4 100644--- a/worker/src/features/evaluation/observationEval/__tests__/observationEval.e2e.test.ts+++ b/worker/src/features/evaluation/observationEval/__tests__/observationEval.e2e.test.ts@@ -328,6 +328,96 @@ describe("Observation Eval E2E Pipeline", () => {);});+ it("records the batch action in the execution metadata of a manual batch run", async () => {+ const observation = createTestObservation({+ project_id: projectId,+ output: '{"response": "ok"}',+ });+ const config = createTestEvalConfig({+ id: `config-${randomUUID()}`,+ projectId,+ variableMapping: [+ { templateVariable: "output", selectedColumnId: "output" },+ ],+ });+ const pipeline = createFullyMockedEvalPipeline({+ observation,+ llmResponse: { score: 1, reasoning: "ok" },+ });+ const batchActionId = `batch-${randomUUID()}`;++ await scheduleObservationEvals({+ observation,+ configs: [config],+ schedulerDeps: pipeline.schedulerDeps,+ executionMode: "MANUAL",+ executionScopeId: batchActionId,+ batchActionId,+ });++ const [enqueued] = (pipeline.schedulerDeps.enqueueEvalJob as Mock).mock+ .calls[0] as Parameters<ObservationEvalSchedulerDeps["enqueueEvalJob"]>;+ expect(enqueued.batchActionId).toBe(batchActionId);++ const mockConfig = createMockJobConfiguration({+ id: config.id,+ projectId,+ evalTemplateId: config.evalTemplateId,+ scoreName: config.scoreName,+ targetObject: EvalTargetObject.EVENT,+ filter: config.filter,+ variableMapping: config.variableMapping,+ evalTemplate: {+ id: config.evalTemplateId,+ projectId,+ name: "Accuracy Evaluator",+ version: 1,+ type: EvalTemplateType.LLM_AS_JUDGE,+ prompt: "Evaluate: {{output}}",+ model: "gpt-4",+ provider: "openai",+ modelParams: {},+ outputDefinition: { score: "0 to 1", reasoning: "Explanation" },+ sourceCode: null,+ sourceCodeLanguage: null,+ vars: ["output"],+ createdAt: new Date(),+ updatedAt: new Date(),+ },+ });+ (prisma.jobExecution.findFirst as Mock).mockResolvedValue({+ id: enqueued.jobExecutionId,+ projectId,+ status: JobExecutionStatus.PENDING,+ jobConfigurationId: config.id,+ jobInputTraceId: observation.trace_id,+ jobInputObservationId: observation.span_id,+ jobInputDatasetItemId: null,+ jobTemplateId: null,+ });+ mockMigratedAssignment(mockConfig);++ await processObservationEval({+ event: {+ projectId,+ jobExecutionId: enqueued.jobExecutionId,+ observationS3Path: enqueued.observationS3Path,+ executionMode: enqueued.executionMode,+ batchActionId: enqueued.batchActionId,+ },+ executionType: EvalTemplateType.LLM_AS_JUDGE,+ deps: pipeline.processorDeps,+ });++ expect(runLLMAsJudgeEvaluation).toHaveBeenCalledWith(+ expect.objectContaining({+ executionMetadata: expect.objectContaining({+ batch_action_id: batchActionId,+ }),+ }),+ );+ });+it("should process a code-based eval through execution and score persistence", async () => {const observation = createTestObservation({project_id: projectId,diff --git a/worker/src/features/evaluation/observationEval/createSchedulerDeps.ts b/worker/src/features/evaluation/observationEval/createSchedulerDeps.tsindex 18f8409cf3..2789ee1e65 100644--- a/worker/src/features/evaluation/observationEval/createSchedulerDeps.ts+++ b/worker/src/features/evaluation/observationEval/createSchedulerDeps.ts@@ -85,6 +85,9 @@ export function createObservationEvalSchedulerDeps(): ObservationEvalSchedulerDe...(params.variableMapping != null? { variableMapping: params.variableMapping }: {}),+ ...(params.batchActionId+ ? { batchActionId: params.batchActionId }+ : {}),};if (params.evalTemplateType === EvalTemplateType.CODE) {diff --git a/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts b/worker/src/features/evaluation/observationEval/observationEvalProcessor.tsindex d4683fa2fa..355ac7059e 100644--- a/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts+++ b/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts@@ -268,6 +268,7 @@ export async function processObservationEval(targetTraceId: job.jobInputTraceId,targetObservationId: job.jobInputObservationId,targetDatasetItemId: job.jobInputDatasetItemId,+ batchActionId: event.batchActionId,}),deps: deps.evalExecutionDeps,};diff --git a/worker/src/features/evaluation/observationEval/scheduleObservationEvals.ts b/worker/src/features/evaluation/observationEval/scheduleObservationEvals.tsindex 9e4d8fbc35..3958042143 100644--- a/worker/src/features/evaluation/observationEval/scheduleObservationEvals.ts+++ b/worker/src/features/evaluation/observationEval/scheduleObservationEvals.ts@@ -44,6 +44,8 @@ interface ScheduleObservationEvalsParams {* stable while a second run gets its own executions.*/executionScopeId?: string;+ /** Batch action of a manual batch run, recorded on each execution. */+ batchActionId?: string;}/**@@ -106,6 +108,7 @@ export async function scheduleObservationEvals(schedulerDeps,executionMode,executionScopeId,+ batchActionId,} = params;// Early return if no configs@@ -193,6 +196,7 @@ export async function scheduleObservationEvals(schedulerDeps,executionMode,executionScopeId,+ batchActionId,}),),);@@ -235,6 +239,7 @@ interface ProcessConfigParams {schedulerDeps: ObservationEvalSchedulerDeps;executionMode?: EvalExecutionMode;executionScopeId?: string;+ batchActionId?: string;}async function processMatchingConfig(@@ -248,6 +253,7 @@ async function processMatchingConfig(schedulerDeps,executionMode,executionScopeId,+ batchActionId,} = params;const jobIdentity: string[] =@@ -304,6 +310,7 @@ async function processMatchingConfig(...(assignment.variableMapping != null? { variableMapping: assignment.variableMapping }: {}),+ ...(batchActionId ? { batchActionId } : {}),});logger.debug("Scheduled observation eval job", {diff --git a/worker/src/features/evaluation/observationEval/types.ts b/worker/src/features/evaluation/observationEval/types.tsindex e156251c1e..209fc58efc 100644--- a/worker/src/features/evaluation/observationEval/types.ts+++ b/worker/src/features/evaluation/observationEval/types.ts@@ -123,5 +123,7 @@ export interface ObservationEvalSchedulerDeps {* evaluator version mapping.*/variableMapping?: ObservationVariableMapping[];+ /** Batch action of a manual batch run; omitted for live ingestion. */+ batchActionId?: string;}) => Promise<void>;}