🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(mcp): Add batchUpsertDatasetItems tool (#17463)
Ben Bachem46630a385589
diff --git a/packages/shared/src/in-app-agent/server/mcpPolicy.ts b/packages/shared/src/in-app-agent/server/mcpPolicy.tsindex 2014dc3d51..6285e319e6 100644--- a/packages/shared/src/in-app-agent/server/mcpPolicy.ts+++ b/packages/shared/src/in-app-agent/server/mcpPolicy.ts@@ -100,6 +100,10 @@ export const IN_APP_AGENT_LANGFUSE_MCP_TOOL_POLICIES = {approval: "approval",availability: { scope: "datasets:CUD" },},+ batchUpsertDatasetItems: {+ approval: "approval",+ availability: { scope: "datasets:CUD" },+ },listDatasetItems: {approval: "auto",availability: { scope: "datasets:read" },diff --git a/web/src/__tests__/server/mcp-public-api-tools.servertest.ts b/web/src/__tests__/server/mcp-public-api-tools.servertest.tsindex 43a3e49780..b166d23954 100644--- a/web/src/__tests__/server/mcp-public-api-tools.servertest.ts+++ b/web/src/__tests__/server/mcp-public-api-tools.servertest.ts@@ -69,6 +69,7 @@ import {handleListComments,} from "@/src/features/mcp/server/comments/tools";import {+ handleBatchUpsertDatasetItems,handleCreateDatasetRunItem,handleDeleteDatasetItem,handleDeleteDatasetRun,@@ -161,6 +162,7 @@ describe("MCP public API tools", () => {"getEvaluationRule","createEvaluationRule","createDashboardWidget",+ "batchUpsertDatasetItems","listDatasets","getHealth","listScores",@@ -252,6 +254,7 @@ describe("MCP public API tools", () => {["addDashboardPlacement","attachEvaluatorToEvaluationRule",+ "batchUpsertDatasetItems","createChatPrompt","createDashboard","createDashboardWidget",@@ -641,6 +644,72 @@ describe("MCP public API tools", () => {)) as { id: string; datasetName: string };expect(datasetItem.datasetName).toBe(renamedDatasetName);+ const firstBatchItemId = uuidv4();+ const batchResult = (await handleBatchUpsertDatasetItems(+ {+ datasetId: dataset.id,+ items: [+ {+ id: firstBatchItemId,+ input: { question: "first batch question" },+ expectedOutput: { answer: "first batch answer" },+ },+ {+ input: { question: "second batch question" },+ expectedOutput: { answer: "second batch answer" },+ },+ ],+ },+ context,+ )) as {+ data: Array<{ id: string; input: unknown; url: string }>;+ };+ expect(batchResult.data).toHaveLength(2);+ expect(batchResult.data[0]).toMatchObject({+ id: firstBatchItemId,+ input: { question: "first batch question" },+ url: expect.stringContaining(firstBatchItemId),+ });++ await expect(+ handleBatchUpsertDatasetItems(+ {+ datasetId: dataset.id,+ items: [+ {+ id: firstBatchItemId,+ input: { question: "updated batch question" },+ },+ ],+ },+ context,+ ),+ ).resolves.toMatchObject({+ data: [+ {+ id: firstBatchItemId,+ input: { question: "updated batch question" },+ },+ ],+ });++ await expect(+ handleBatchUpsertDatasetItems(+ { datasetId: dataset.id, items: [] },+ context,+ ),+ ).rejects.toThrow("Validation failed");++ await expect(+ handleBatchUpsertDatasetItems(+ {+ datasetId: dataset.id,+ items: Array.from({ length: 101 }, () => ({ input: {} })),+ },+ context,+ ),+ ).rejects.toThrow("Validation failed");+const datasetItems = (await handleListDatasetItems({ datasetId: dataset.id, page: 1, limit: 10 },context,diff --git a/web/src/features/in-app-agent/components/utils/side-effects.ts b/web/src/features/in-app-agent/components/utils/side-effects.tsindex 93df4c27d0..1ccef24360 100644--- a/web/src/features/in-app-agent/components/utils/side-effects.ts+++ b/web/src/features/in-app-agent/components/utils/side-effects.ts@@ -59,6 +59,7 @@ const IN_APP_AGENT_TOOL_TRPC_INVALIDATION_TARGETS = {langfuse_listDatasets: [],langfuse_getDataset: [],langfuse_upsertDatasetItem: ["datasets"],+ langfuse_batchUpsertDatasetItems: ["datasets"],langfuse_listDatasetItems: [],langfuse_getDatasetItem: [],langfuse_deleteDatasetItem: ["datasets"],diff --git a/web/src/features/in-app-agent/components/utils/utils.ts b/web/src/features/in-app-agent/components/utils/utils.tsindex 8199863a31..f26c33251e 100644--- a/web/src/features/in-app-agent/components/utils/utils.ts+++ b/web/src/features/in-app-agent/components/utils/utils.ts@@ -53,6 +53,7 @@ export function getInAppAgentToolDisplayName(toolName: string): string {const IN_APP_AGENT_TOOL_PROGRESS_LABEL_OVERRIDES: Record<string, string> = {addDashboardPlacement: "Adding widget to dashboard",bash: "Running command",+ batchUpsertDatasetItems: "Saving dataset items",createAnnotationQueueAssignment: "Assigning annotation queue",createAnnotationQueueItem: "Adding to annotation queue",createChatPrompt: "Creating chat prompt",diff --git a/web/src/features/mcp/server/datasets/index.ts b/web/src/features/mcp/server/datasets/index.tsindex 821f2a143c..2bd85b8c6a 100644--- a/web/src/features/mcp/server/datasets/index.ts+++ b/web/src/features/mcp/server/datasets/index.ts@@ -4,6 +4,10 @@ import {upsertDatasetItemTool,handleUpsertDatasetItem,} from "./tools/upsertDatasetItem";+import {+ batchUpsertDatasetItemsTool,+ handleBatchUpsertDatasetItems,+} from "./tools/batchUpsertDatasetItems";import {createDatasetRunItemTool,handleCreateDatasetRunItem,@@ -51,6 +55,10 @@ export const datasetsFeature = {handler: handleGetDataset,},{ definition: upsertDatasetItemTool, handler: handleUpsertDatasetItem },+ {+ definition: batchUpsertDatasetItemsTool,+ handler: handleBatchUpsertDatasetItems,+ },{definition: listDatasetItemsTool,handler: handleListDatasetItems,diff --git a/web/src/features/mcp/server/datasets/schema.ts b/web/src/features/mcp/server/datasets/schema.tsindex d6a05c28a7..b0a7241ac1 100644--- a/web/src/features/mcp/server/datasets/schema.ts+++ b/web/src/features/mcp/server/datasets/schema.ts@@ -59,8 +59,7 @@ export const GetDatasetItemsMcpInput = GetDatasetItemsMcpRuntimeInput.refine(},);-export const PostDatasetItemMcpInput = z.object({- datasetId: z.string(),+const DatasetItemMcpInput = z.object({input: z.any().optional(),expectedOutput: z.any().optional(),metadata: z.any().optional(),@@ -70,4 +69,13 @@ export const PostDatasetItemMcpInput = z.object({status: z.enum(["ACTIVE", "ARCHIVED"]).optional(),});+export const PostDatasetItemMcpInput = DatasetItemMcpInput.extend({+ datasetId: z.string(),+});++export const BatchUpsertDatasetItemsMcpInput = z.object({+ datasetId: z.string(),+ items: z.array(DatasetItemMcpInput).min(1).max(100),+});+export const GetDatasetItemsMcpBaseSchema = GetDatasetItemsMcpBaseInput;diff --git a/web/src/features/mcp/server/datasets/tools/batchUpsertDatasetItems.ts b/web/src/features/mcp/server/datasets/tools/batchUpsertDatasetItems.tsnew file mode 100644index 0000000000..e93938a8d6--- /dev/null+++ b/web/src/features/mcp/server/datasets/tools/batchUpsertDatasetItems.ts@@ -0,0 +1,49 @@+import { createDatasetItemForApi } from "@/src/features/datasets/server";+import { PostDatasetItemsV1Response } from "@/src/features/public-api/server";+import { buildDatasetItemUrl } from "@langfuse/shared/src/server";+import { defineTool } from "../../../core/define-tool";+import { runMcpTool } from "../../../core/run-mcp-tool";+import { BatchUpsertDatasetItemsMcpInput } from "../schema";++export const [batchUpsertDatasetItemsTool, handleBatchUpsertDatasetItems] =+ defineTool({+ name: "batchUpsertDatasetItems",+ description:+ "Upsert multiple dataset items in one dataset. Items are processed in order. The operation stops on the first failure and is not atomic.",+ baseSchema: BatchUpsertDatasetItemsMcpInput,+ inputSchema: BatchUpsertDatasetItemsMcpInput,+ handler: async (input, context) =>+ runMcpTool({+ spanName: "mcp.dataset_items.batch_upsert",+ context,+ attributes: {+ "mcp.dataset_id": input.datasetId,+ "mcp.dataset_item_count": input.items.length,+ },+ fn: async () => {+ const data = [];++ for (const item of input.items) {+ const result = await createDatasetItemForApi({+ input: { ...item, datasetId: input.datasetId },+ projectId: context.projectId,+ auditScope: context,+ });+ const datasetItem = PostDatasetItemsV1Response.parse(result);++ data.push({+ ...datasetItem,+ url: buildDatasetItemUrl({+ projectId: context.projectId,+ datasetId: input.datasetId,+ datasetItemId: datasetItem.id,+ }),+ });+ }++ return { data };+ },+ }),+ destructiveHint: true,+ expensiveHint: true,+ });diff --git a/web/src/features/mcp/server/datasets/tools/index.ts b/web/src/features/mcp/server/datasets/tools/index.tsindex 46829eb522..92af6584a2 100644--- a/web/src/features/mcp/server/datasets/tools/index.ts+++ b/web/src/features/mcp/server/datasets/tools/index.ts@@ -1,5 +1,6 @@export * from "./upsertDataset";export * from "./upsertDatasetItem";+export * from "./batchUpsertDatasetItems";export * from "./createDatasetRunItem";export * from "./deleteDatasetItem";export * from "./deleteDatasetRun";