Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(mcp): Add batchUpsertDatasetItems tool (#17463)

Ben Bachem46630a385589
diff --git a/packages/shared/src/in-app-agent/server/mcpPolicy.ts b/packages/shared/src/in-app-agent/server/mcpPolicy.ts
index 2014dc3d51..6285e319e6 100644
--- a/packages/shared/src/in-app-agent/server/mcpPolicy.ts
+++ b/packages/shared/src/in-app-agent/server/mcpPolicy.ts
@@ -100,6 +100,10 @@ export const IN_APP_AGENT_LANGFUSE_MCP_TOOL_POLICIES = {
approval: "approval",
availability: { scope: "datasets:CUD" },
},
+ batchUpsertDatasetItems: {
+ approval: "approval",
+ availability: { scope: "datasets:CUD" },
+ },
listDatasetItems: {
approval: "auto",
availability: { scope: "datasets:read" },
diff --git a/web/src/__tests__/server/mcp-public-api-tools.servertest.ts b/web/src/__tests__/server/mcp-public-api-tools.servertest.ts
index 43a3e49780..b166d23954 100644
--- a/web/src/__tests__/server/mcp-public-api-tools.servertest.ts
+++ b/web/src/__tests__/server/mcp-public-api-tools.servertest.ts
@@ -69,6 +69,7 @@ import {
handleListComments,
} from "@/src/features/mcp/server/comments/tools";
import {
+ handleBatchUpsertDatasetItems,
handleCreateDatasetRunItem,
handleDeleteDatasetItem,
handleDeleteDatasetRun,
@@ -161,6 +162,7 @@ describe("MCP public API tools", () => {
"getEvaluationRule",
"createEvaluationRule",
"createDashboardWidget",
+ "batchUpsertDatasetItems",
"listDatasets",
"getHealth",
"listScores",
@@ -252,6 +254,7 @@ describe("MCP public API tools", () => {
[
"addDashboardPlacement",
"attachEvaluatorToEvaluationRule",
+ "batchUpsertDatasetItems",
"createChatPrompt",
"createDashboard",
"createDashboardWidget",
@@ -641,6 +644,72 @@ describe("MCP public API tools", () => {
)) as { id: string; datasetName: string };
expect(datasetItem.datasetName).toBe(renamedDatasetName);
+ const firstBatchItemId = uuidv4();
+ const batchResult = (await handleBatchUpsertDatasetItems(
+ {
+ datasetId: dataset.id,
+ items: [
+ {
+ id: firstBatchItemId,
+ input: { question: "first batch question" },
+ expectedOutput: { answer: "first batch answer" },
+ },
+ {
+ input: { question: "second batch question" },
+ expectedOutput: { answer: "second batch answer" },
+ },
+ ],
+ },
+ context,
+ )) as {
+ data: Array<{ id: string; input: unknown; url: string }>;
+ };
+ expect(batchResult.data).toHaveLength(2);
+ expect(batchResult.data[0]).toMatchObject({
+ id: firstBatchItemId,
+ input: { question: "first batch question" },
+ url: expect.stringContaining(firstBatchItemId),
+ });
+
+ await expect(
+ handleBatchUpsertDatasetItems(
+ {
+ datasetId: dataset.id,
+ items: [
+ {
+ id: firstBatchItemId,
+ input: { question: "updated batch question" },
+ },
+ ],
+ },
+ context,
+ ),
+ ).resolves.toMatchObject({
+ data: [
+ {
+ id: firstBatchItemId,
+ input: { question: "updated batch question" },
+ },
+ ],
+ });
+
+ await expect(
+ handleBatchUpsertDatasetItems(
+ { datasetId: dataset.id, items: [] },
+ context,
+ ),
+ ).rejects.toThrow("Validation failed");
+
+ await expect(
+ handleBatchUpsertDatasetItems(
+ {
+ datasetId: dataset.id,
+ items: Array.from({ length: 101 }, () => ({ input: {} })),
+ },
+ context,
+ ),
+ ).rejects.toThrow("Validation failed");
+
const datasetItems = (await handleListDatasetItems(
{ datasetId: dataset.id, page: 1, limit: 10 },
context,
diff --git a/web/src/features/in-app-agent/components/utils/side-effects.ts b/web/src/features/in-app-agent/components/utils/side-effects.ts
index 93df4c27d0..1ccef24360 100644
--- a/web/src/features/in-app-agent/components/utils/side-effects.ts
+++ b/web/src/features/in-app-agent/components/utils/side-effects.ts
@@ -59,6 +59,7 @@ const IN_APP_AGENT_TOOL_TRPC_INVALIDATION_TARGETS = {
langfuse_listDatasets: [],
langfuse_getDataset: [],
langfuse_upsertDatasetItem: ["datasets"],
+ langfuse_batchUpsertDatasetItems: ["datasets"],
langfuse_listDatasetItems: [],
langfuse_getDatasetItem: [],
langfuse_deleteDatasetItem: ["datasets"],
diff --git a/web/src/features/in-app-agent/components/utils/utils.ts b/web/src/features/in-app-agent/components/utils/utils.ts
index 8199863a31..f26c33251e 100644
--- a/web/src/features/in-app-agent/components/utils/utils.ts
+++ b/web/src/features/in-app-agent/components/utils/utils.ts
@@ -53,6 +53,7 @@ export function getInAppAgentToolDisplayName(toolName: string): string {
const IN_APP_AGENT_TOOL_PROGRESS_LABEL_OVERRIDES: Record<string, string> = {
addDashboardPlacement: "Adding widget to dashboard",
bash: "Running command",
+ batchUpsertDatasetItems: "Saving dataset items",
createAnnotationQueueAssignment: "Assigning annotation queue",
createAnnotationQueueItem: "Adding to annotation queue",
createChatPrompt: "Creating chat prompt",
diff --git a/web/src/features/mcp/server/datasets/index.ts b/web/src/features/mcp/server/datasets/index.ts
index 821f2a143c..2bd85b8c6a 100644
--- a/web/src/features/mcp/server/datasets/index.ts
+++ b/web/src/features/mcp/server/datasets/index.ts
@@ -4,6 +4,10 @@ import {
upsertDatasetItemTool,
handleUpsertDatasetItem,
} from "./tools/upsertDatasetItem";
+import {
+ batchUpsertDatasetItemsTool,
+ handleBatchUpsertDatasetItems,
+} from "./tools/batchUpsertDatasetItems";
import {
createDatasetRunItemTool,
handleCreateDatasetRunItem,
@@ -51,6 +55,10 @@ export const datasetsFeature = {
handler: handleGetDataset,
},
{ definition: upsertDatasetItemTool, handler: handleUpsertDatasetItem },
+ {
+ definition: batchUpsertDatasetItemsTool,
+ handler: handleBatchUpsertDatasetItems,
+ },
{
definition: listDatasetItemsTool,
handler: handleListDatasetItems,
diff --git a/web/src/features/mcp/server/datasets/schema.ts b/web/src/features/mcp/server/datasets/schema.ts
index d6a05c28a7..b0a7241ac1 100644
--- a/web/src/features/mcp/server/datasets/schema.ts
+++ b/web/src/features/mcp/server/datasets/schema.ts
@@ -59,8 +59,7 @@ export const GetDatasetItemsMcpInput = GetDatasetItemsMcpRuntimeInput.refine(
},
);
-export const PostDatasetItemMcpInput = z.object({
- datasetId: z.string(),
+const DatasetItemMcpInput = z.object({
input: z.any().optional(),
expectedOutput: z.any().optional(),
metadata: z.any().optional(),
@@ -70,4 +69,13 @@ export const PostDatasetItemMcpInput = z.object({
status: z.enum(["ACTIVE", "ARCHIVED"]).optional(),
});
+export const PostDatasetItemMcpInput = DatasetItemMcpInput.extend({
+ datasetId: z.string(),
+});
+
+export const BatchUpsertDatasetItemsMcpInput = z.object({
+ datasetId: z.string(),
+ items: z.array(DatasetItemMcpInput).min(1).max(100),
+});
+
export const GetDatasetItemsMcpBaseSchema = GetDatasetItemsMcpBaseInput;
diff --git a/web/src/features/mcp/server/datasets/tools/batchUpsertDatasetItems.ts b/web/src/features/mcp/server/datasets/tools/batchUpsertDatasetItems.ts
new file mode 100644
index 0000000000..e93938a8d6
--- /dev/null
+++ b/web/src/features/mcp/server/datasets/tools/batchUpsertDatasetItems.ts
@@ -0,0 +1,49 @@
+import { createDatasetItemForApi } from "@/src/features/datasets/server";
+import { PostDatasetItemsV1Response } from "@/src/features/public-api/server";
+import { buildDatasetItemUrl } from "@langfuse/shared/src/server";
+import { defineTool } from "../../../core/define-tool";
+import { runMcpTool } from "../../../core/run-mcp-tool";
+import { BatchUpsertDatasetItemsMcpInput } from "../schema";
+
+export const [batchUpsertDatasetItemsTool, handleBatchUpsertDatasetItems] =
+ defineTool({
+ name: "batchUpsertDatasetItems",
+ description:
+ "Upsert multiple dataset items in one dataset. Items are processed in order. The operation stops on the first failure and is not atomic.",
+ baseSchema: BatchUpsertDatasetItemsMcpInput,
+ inputSchema: BatchUpsertDatasetItemsMcpInput,
+ handler: async (input, context) =>
+ runMcpTool({
+ spanName: "mcp.dataset_items.batch_upsert",
+ context,
+ attributes: {
+ "mcp.dataset_id": input.datasetId,
+ "mcp.dataset_item_count": input.items.length,
+ },
+ fn: async () => {
+ const data = [];
+
+ for (const item of input.items) {
+ const result = await createDatasetItemForApi({
+ input: { ...item, datasetId: input.datasetId },
+ projectId: context.projectId,
+ auditScope: context,
+ });
+ const datasetItem = PostDatasetItemsV1Response.parse(result);
+
+ data.push({
+ ...datasetItem,
+ url: buildDatasetItemUrl({
+ projectId: context.projectId,
+ datasetId: input.datasetId,
+ datasetItemId: datasetItem.id,
+ }),
+ });
+ }
+
+ return { data };
+ },
+ }),
+ destructiveHint: true,
+ expensiveHint: true,
+ });
diff --git a/web/src/features/mcp/server/datasets/tools/index.ts b/web/src/features/mcp/server/datasets/tools/index.ts
index 46829eb522..92af6584a2 100644
--- a/web/src/features/mcp/server/datasets/tools/index.ts
+++ b/web/src/features/mcp/server/datasets/tools/index.ts
@@ -1,5 +1,6 @@
export * from "./upsertDataset";
export * from "./upsertDatasetItem";
+export * from "./batchUpsertDatasetItems";
export * from "./createDatasetRunItem";
export * from "./deleteDatasetItem";
export * from "./deleteDatasetRun";