🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
vi.hoisted(() => {
process.env.LANGFUSE_MIGRATION_V4_WRITE_MODE = "dual";
});
process.env.LANGFUSE_DATASET_SERVICE_READ_FROM_VERSIONED_IMPLEMENTATION =
"true";
process.env.LANGFUSE_DATASET_SERVICE_WRITE_TO_VERSIONED_IMPLEMENTATION = "true";
vi.mock("@langfuse/shared/src/server", async () => {
const actual = await vi.importActual("@langfuse/shared/src/server");
const queue = {
add: vi.fn().mockResolvedValue(undefined),
disconnect: vi.fn(),
on: vi.fn(),
};
return {
...actual,
EventPropagationQueue: {
getInstance: () => queue,
},
EntityChangeQueue: {
getInstance: () => queue,
},
DatasetRunItemUpsertQueue: {
getInstance: () => queue,
},
DatasetDeleteQueue: {
getInstance: () => queue,
},
};
});
import { v4 as uuidv4 } from "uuid";
import { prisma } from "@langfuse/shared/src/db";
import {
createObservation,
createObservationsCh,
createDatasetRunItem,
createDatasetRunItemsCh,
createTrace,
createTracesCh,
} from "@langfuse/shared/src/server";
import {
createMcpTestSetup,
createPromptInDb,
mockServerContext,
verifyAuditLog,
waitFor,
} from "./mcp-helpers";
import "@/src/features/mcp/server/bootstrap";
import { config as mcpRouteConfig } from "@/src/pages/api/public/mcp";
import { toolRegistry } from "@/src/features/mcp/server/registry";
import { handleCreateAnnotationQueue } from "@/src/features/mcp/server/annotationQueues/tools/createAnnotationQueue";
import { handleCreateAnnotationQueueAssignment } from "@/src/features/mcp/server/annotationQueues/tools/createAnnotationQueueAssignment";
import { handleCreateAnnotationQueueItem } from "@/src/features/mcp/server/annotationQueues/tools/createAnnotationQueueItem";
import { handleDeleteAnnotationQueueAssignment } from "@/src/features/mcp/server/annotationQueues/tools/deleteAnnotationQueueAssignment";
import { handleDeleteAnnotationQueueItem } from "@/src/features/mcp/server/annotationQueues/tools/deleteAnnotationQueueItem";
import { handleGetAnnotationQueue } from "@/src/features/mcp/server/annotationQueues/tools/getAnnotationQueue";
import { handleGetAnnotationQueueItem } from "@/src/features/mcp/server/annotationQueues/tools/getAnnotationQueueItem";
import { handleListAnnotationQueueItems } from "@/src/features/mcp/server/annotationQueues/tools/listAnnotationQueueItems";
import { handleListAnnotationQueues } from "@/src/features/mcp/server/annotationQueues/tools/listAnnotationQueues";
import { handleUpdateAnnotationQueueItem } from "@/src/features/mcp/server/annotationQueues/tools/updateAnnotationQueueItem";
import { handleCreateComment } from "@/src/features/mcp/server/comments/tools/createComment";
import { handleGetComment } from "@/src/features/mcp/server/comments/tools/getComment";
import { handleListComments } from "@/src/features/mcp/server/comments/tools/listComments";
import { handleBatchUpsertDatasetItems } from "@/src/features/mcp/server/datasets/tools/batchUpsertDatasetItems";
import { handleCreateDatasetRunItem } from "@/src/features/mcp/server/datasets/tools/createDatasetRunItem";
import { handleDeleteDatasetItem } from "@/src/features/mcp/server/datasets/tools/deleteDatasetItem";
import { handleDeleteDatasetRun } from "@/src/features/mcp/server/datasets/tools/deleteDatasetRun";
import { handleGetDataset } from "@/src/features/mcp/server/datasets/tools/getDataset";
import { handleGetDatasetItem } from "@/src/features/mcp/server/datasets/tools/getDatasetItem";
import { handleGetDatasetRun } from "@/src/features/mcp/server/datasets/tools/getDatasetRun";
import { handleListDatasetItems } from "@/src/features/mcp/server/datasets/tools/listDatasetItems";
import { handleListDatasetRunItems } from "@/src/features/mcp/server/datasets/tools/listDatasetRunItems";
import { handleListDatasetRuns } from "@/src/features/mcp/server/datasets/tools/listDatasetRuns";
import { handleListDatasets } from "@/src/features/mcp/server/datasets/tools/listDatasets";
import {
handleUpsertDataset,
upsertDatasetTool,
} from "@/src/features/mcp/server/datasets/tools/upsertDataset";
import { handleUpsertDatasetItem } from "@/src/features/mcp/server/datasets/tools/upsertDatasetItem";
import { handleGetHealth } from "@/src/features/mcp/server/health/tools/getHealth";
import { handleCreateModel } from "@/src/features/mcp/server/models/tools/createModel";
import { handleDeleteModel } from "@/src/features/mcp/server/models/tools/deleteModel";
import { handleGetModel } from "@/src/features/mcp/server/models/tools/getModel";
import { handleListModels } from "@/src/features/mcp/server/models/tools/listModels";
import { handleCreateScoreConfig } from "@/src/features/mcp/server/scores/tools/createScoreConfig";
import { handleGetScoreConfig } from "@/src/features/mcp/server/scores/tools/getScoreConfig";
import { handleListScoreConfigs } from "@/src/features/mcp/server/scores/tools/listScoreConfigs";
import { handleUpdateScoreConfig } from "@/src/features/mcp/server/scores/tools/updateScoreConfig";
const createScoreConfig = async (projectId: string) =>
prisma.scoreConfig.create({
data: {
id: uuidv4(),
projectId,
name: `mcp-score-${uuidv4().slice(0, 8)}`,
dataType: "NUMERIC",
},
});
const createProjectUser = async ({
orgId,
projectId,
}: {
orgId: string;
projectId: string;
}) => {
const user = await prisma.user.create({
data: {
id: uuidv4(),
email: `mcp-user-${uuidv4()}@example.com`,
name: "MCP Test User",
},
});
const orgMembership = await prisma.organizationMembership.create({
data: {
orgId,
userId: user.id,
role: "MEMBER",
},
});
await prisma.projectMembership.create({
data: {
projectId,
userId: user.id,
role: "MEMBER",
orgMembershipId: orgMembership.id,
},
});
return user;
};
describe("MCP public API tools", () => {
const getToolNames = async (context = mockServerContext()) =>
(await toolRegistry.getToolDefinitions(context)).map((tool) => tool.name);
it("registers public API tools", async () => {
const toolNames = await getToolNames();
expect(toolNames).toEqual(
expect.arrayContaining([
"listAnnotationQueues",
"createAnnotationQueue",
"createComment",
"listEvaluators",
"listManagedEvaluatorTemplates",
"getEvaluator",
"createEvaluator",
"updateEvaluator",
"listEvaluationRules",
"getEvaluationRule",
"createEvaluationRule",
"createDashboardWidget",
"batchUpsertDatasetItems",
"listDatasets",
"getHealth",
"listScores",
"getScore",
"listAlerts",
"getAlert",
"createModel",
"createScoreConfig",
]),
);
});
it("exposes the same feature-enabled tools for in-app agent keys", async () => {
const toolNames = await getToolNames();
const inAppToolNames = await getToolNames(
mockServerContext({ inAppAgent: { permissions: "read" } }),
);
expect(inAppToolNames.sort()).toEqual(toolNames.sort());
});
it("does not resolve mutating tools for in-app agent keys without a run override", async () => {
const context = mockServerContext({
inAppAgent: { permissions: "read" },
});
const inAppToolNames = await getToolNames(context);
expect(inAppToolNames).toEqual(
expect.arrayContaining(["upsertDataset", "createModel"]),
);
await expect(
toolRegistry.getEnabledTool("upsertDataset", context),
).resolves.toBeUndefined();
await expect(
toolRegistry.getEnabledTool("createModel", context),
).resolves.toBeUndefined();
await expect(
toolRegistry.getEnabledTool("createDashboardWidget", context),
).resolves.toBeUndefined();
});
it("resolves allowlisted mutating tools plus reads for in-app agent keys", async () => {
const context = mockServerContext({
inAppAgent: {
permissions: "tool-allowlist",
allowedToolNames: ["upsertDataset"],
},
});
await expect(
toolRegistry.getEnabledTool("upsertDataset", context),
).resolves.toBeTruthy();
await expect(
toolRegistry.getEnabledTool("createModel", context),
).resolves.toBeUndefined();
await expect(
toolRegistry.getEnabledTool("listDatasets", context),
).resolves.toBeTruthy();
});
it("resolves the dashboard widget creation override for in-app agent keys", async () => {
const context = mockServerContext({
inAppAgent: {
permissions: "tool-allowlist",
allowedToolNames: ["createDashboardWidget"],
},
});
await expect(
toolRegistry.getEnabledTool("createDashboardWidget", context),
).resolves.toBeTruthy();
await expect(
toolRegistry.getEnabledTool("upsertDataset", context),
).resolves.toBeUndefined();
});
it("marks destructive public API tools", async () => {
const toolNames = await getToolNames();
const destructiveToolNames = toolNames
.filter(
(toolName) =>
toolRegistry.getTool(toolName)?.definition.annotations
?.destructiveHint,
)
.sort();
expect(destructiveToolNames).toEqual(
[
"addDashboardPlacement",
"attachEvaluatorToEvaluationRule",
"batchUpsertDatasetItems",
"createChatPrompt",
"createDashboard",
"createDashboardWidget",
"createEvaluationRule",
"createEvaluator",
"createScore",
"createScoreConfig",
"createSkillVersion",
"createTextPrompt",
"deleteAnnotationQueueAssignment",
"deleteAnnotationQueueItem",
"deleteDashboard",
"deleteDashboardPlacement",
"deleteDashboardWidget",
"deleteDatasetItem",
"deleteDatasetRun",
"deleteEvaluationRule",
"deleteEvaluator",
"deleteModel",
"deleteScoreConfig",
"detachEvaluatorFromEvaluationRule",
"updateAnnotationQueueItem",
"updateDashboard",
"updateDashboardPlacement",
"updateDashboardWidget",
"updateEvaluationRule",
"updateEvaluator",
"updatePromptLabels",
"updateScoreConfig",
"updateSkillLabels",
"updateSkillTags",
"upsertDataset",
"upsertDatasetItem",
].sort(),
);
});
it("uses a larger MCP request body limit", () => {
expect(mcpRouteConfig.api.bodyParser.sizeLimit).toBe("4.5mb");
});
it("covers annotation queue public API routes", async () => {
const { context, projectId, orgId } = await createMcpTestSetup();
const scoreConfig = await createScoreConfig(projectId);
const user = await createProjectUser({ orgId, projectId });
const queue = (await handleCreateAnnotationQueue(
{
name: `mcp-queue-${uuidv4()}`,
description: "MCP queue",
scoreConfigIds: [scoreConfig.id],
},
context,
)) as { id: string; name: string };
const queues = (await handleListAnnotationQueues(
{ page: 1, limit: 10 },
context,
)) as { data: Array<{ id: string }> };
expect(queues.data.map((item) => item.id)).toContain(queue.id);
await expect(
handleGetAnnotationQueue({ queueId: queue.id }, context),
).resolves.toMatchObject({ id: queue.id, name: queue.name });
const queueItem = (await handleCreateAnnotationQueueItem(
{
queueId: queue.id,
objectId: uuidv4(),
objectType: "TRACE",
} as unknown as Parameters<typeof handleCreateAnnotationQueueItem>[0],
context,
)) as { id: string; status: string };
expect(queueItem.status).toBe("PENDING");
const queueItems = (await handleListAnnotationQueueItems(
{ queueId: queue.id, page: 1, limit: 10 },
context,
)) as { data: Array<{ id: string }> };
expect(queueItems.data.map((item) => item.id)).toContain(queueItem.id);
await expect(
handleGetAnnotationQueueItem(
{ queueId: queue.id, itemId: queueItem.id },
context,
),
).resolves.toMatchObject({ id: queueItem.id, queueId: queue.id });
await expect(
handleUpdateAnnotationQueueItem(
{ queueId: queue.id, itemId: queueItem.id, status: "COMPLETED" },
context,
),
).resolves.toMatchObject({ id: queueItem.id, status: "COMPLETED" });
await expect(
handleCreateAnnotationQueueAssignment(
{ queueId: queue.id, userId: user.id },
context,
),
).resolves.toMatchObject({
queueId: queue.id,
userId: user.id,
projectId,
});
const assignment = await prisma.annotationQueueAssignment.findUniqueOrThrow(
{
where: {
projectId_queueId_userId: {
projectId,
queueId: queue.id,
userId: user.id,
},
},
},
);
const assignmentAuditLogCount = await prisma.auditLog.count({
where: {
resourceType: "annotationQueueAssignment",
resourceId: assignment.id,
action: "create",
},
});
await expect(
handleCreateAnnotationQueueAssignment(
{ queueId: queue.id, userId: user.id },
context,
),
).resolves.toMatchObject({
queueId: queue.id,
userId: user.id,
projectId,
});
// Assignment creation uses an upsert for public API parity, so duplicate
// calls are audited even when the assignment already exists.
await expect(
prisma.auditLog.count({
where: {
resourceType: "annotationQueueAssignment",
resourceId: assignment.id,
action: "create",
},
}),
).resolves.toBe(assignmentAuditLogCount + 1);
const auditLogCreateSpy = vi
.spyOn(prisma.auditLog, "create")
.mockRejectedValueOnce(new Error("audit failed"));
try {
await expect(
handleDeleteAnnotationQueueAssignment(
{ queueId: queue.id, userId: user.id },
context,
),
).rejects.toThrow("audit failed");
} finally {
auditLogCreateSpy.mockRestore();
}
await handleCreateAnnotationQueueAssignment(
{ queueId: queue.id, userId: user.id },
context,
);
await expect(
handleDeleteAnnotationQueueAssignment(
{ queueId: queue.id, userId: user.id },
context,
),
).resolves.toEqual({ success: true });
await expect(
handleDeleteAnnotationQueueAssignment(
{ queueId: queue.id, userId: user.id },
context,
),
).rejects.toThrow("Annotation queue assignment not found");
await expect(
handleDeleteAnnotationQueueItem(
{ queueId: queue.id, itemId: queueItem.id },
context,
),
).resolves.toEqual({
success: true,
message: "Annotation queue item deleted successfully",
});
});
it("covers comment and model public API routes", async () => {
const { context, projectId } = await createMcpTestSetup();
const prompt = await createPromptInDb({
projectId,
name: `mcp-comment-prompt-${uuidv4()}`,
prompt: "Test prompt",
});
const created = (await handleCreateComment(
{
objectType: "PROMPT",
objectId: prompt.id,
content: "MCP comment",
},
context,
)) as { id: string };
const listed = (await handleListComments(
{
objectType: "PROMPT",
objectId: prompt.id,
page: 1,
limit: 10,
},
context,
)) as { data: Array<{ id: string; content: string }> };
expect(listed.data).toEqual(
expect.arrayContaining([
expect.objectContaining({ id: created.id, content: "MCP comment" }),
]),
);
await expect(
handleGetComment({ commentId: created.id }, context),
).resolves.toMatchObject({ id: created.id, objectId: prompt.id });
const modelName = `0000-mcp-model-${uuidv4()}`;
const model = (await handleCreateModel(
{
modelName,
matchPattern: `(?i)^${modelName}$`,
unit: "TOKENS",
inputPrice: 0.001,
outputPrice: 0.002,
},
context,
)) as { id: string; modelName: string };
expect(model.modelName).toBe(modelName);
const models = (await handleListModels(
{ page: 1, limit: 100 },
context,
)) as { data: Array<{ id: string }> };
expect(models.data.map((item) => item.id)).toContain(model.id);
await expect(
handleGetModel({ modelId: model.id }, context),
).resolves.toMatchObject({ id: model.id, modelName });
await expect(
handleDeleteModel({ modelId: model.id }, context),
).resolves.toEqual({ message: "Model successfully deleted" });
});
it("covers dataset, dataset item, run item, and run public API routes", async () => {
const { context, projectId, apiKeyId } = await createMcpTestSetup();
const datasetName = `mcp-dataset-100% accuracy %20 ${uuidv4()}`;
const traceId = uuidv4();
const observationId = uuidv4();
await createTracesCh([
createTrace({
id: traceId,
name: "mcp-dataset-trace",
user_id: "mcp-user",
project_id: projectId,
}),
]);
await createObservationsCh([
createObservation({
id: observationId,
trace_id: traceId,
project_id: projectId,
type: "GENERATION",
name: "mcp-dataset-generation",
start_time: new Date("2026-01-01T00:00:00.000Z").getTime(),
end_time: new Date("2026-01-01T00:00:01.000Z").getTime(),
}),
]);
const datasetInputSchema = {
type: "object",
properties: { question: { type: "string" } },
required: ["question"],
additionalProperties: false,
};
const datasetExpectedOutputSchema = {
type: ["object", "null"],
properties: { answer: { type: "string" } },
required: ["answer"],
additionalProperties: false,
};
const upsertDataset = await toolRegistry.getEnabledTool(
upsertDatasetTool.name,
context,
);
expect(upsertDataset).toBeDefined();
const dataset = (await upsertDataset?.handler(
{
name: datasetName,
description: "MCP dataset",
metadata: { source: "mcp" },
inputSchema: datasetInputSchema,
expectedOutputSchema: datasetExpectedOutputSchema,
},
context,
)) as {
id: string;
name: string;
inputSchema: unknown;
expectedOutputSchema: unknown;
};
expect(dataset.name).toBe(datasetName);
expect(dataset.inputSchema).toEqual(datasetInputSchema);
expect(dataset.expectedOutputSchema).toEqual(datasetExpectedOutputSchema);
await expect(
upsertDataset?.handler(
{
name: datasetName,
inputSchema: JSON.stringify(datasetInputSchema),
expectedOutputSchema: JSON.stringify(datasetExpectedOutputSchema),
},
context,
),
).resolves.toMatchObject({
id: dataset.id,
inputSchema: datasetInputSchema,
expectedOutputSchema: datasetExpectedOutputSchema,
});
const renamedDatasetName = `mcp-dataset-renamed-${uuidv4()}`;
const renamedDataset = (await handleUpsertDataset(
{
id: dataset.id,
name: renamedDatasetName,
description: "Renamed MCP dataset",
},
context,
)) as { id: string; name: string; description: string };
expect(renamedDataset).toMatchObject({
id: dataset.id,
name: renamedDatasetName,
description: "Renamed MCP dataset",
});
await expect(
handleUpsertDataset(
{
id: "",
name: `mcp-dataset-empty-id-${uuidv4()}`,
},
context,
),
).rejects.toMatchObject({ name: "ZodError" });
const conflictingDatasetName = `mcp-dataset-conflict-${uuidv4()}`;
await handleUpsertDataset({ name: conflictingDatasetName }, context);
await expect(
handleUpsertDataset(
{
id: dataset.id,
name: conflictingDatasetName,
},
context,
),
).rejects.toThrow("Dataset name already in use");
const datasets = (await handleListDatasets(
{ page: 1, limit: 10 },
context,
)) as { data: Array<{ id: string }> };
expect(datasets.data.map((item) => item.id)).toContain(dataset.id);
await expect(
handleGetDataset({ datasetId: dataset.id }, context),
).resolves.toMatchObject({ id: dataset.id, name: renamedDatasetName });
const datasetItem = (await handleUpsertDatasetItem(
{
datasetId: dataset.id,
input: { question: "ping" },
expectedOutput: { answer: "pong" },
},
context,
)) as { id: string; datasetName: string };
expect(datasetItem.datasetName).toBe(renamedDatasetName);
await expect(
handleUpsertDatasetItem(
{
datasetId: dataset.id,
id: datasetItem.id,
expectedOutput: null,
},
context,
),
).resolves.toMatchObject({ expectedOutput: null });
const firstBatchItemId = uuidv4();
const batchResult = (await handleBatchUpsertDatasetItems(
{
datasetId: dataset.id,
items: [
{
id: firstBatchItemId,
input: { question: "first batch question" },
expectedOutput: { answer: "first batch answer" },
},
{
input: { question: "second batch question" },
expectedOutput: { answer: "second batch answer" },
},
],
},
context,
)) as {
data: Array<{ id: string; input: unknown; url: string }>;
};
expect(batchResult.data).toHaveLength(2);
expect(batchResult.data[0]).toMatchObject({
id: firstBatchItemId,
input: { question: "first batch question" },
url: expect.stringContaining(firstBatchItemId),
});
await expect(
handleBatchUpsertDatasetItems(
{
datasetId: dataset.id,
items: [
{
id: firstBatchItemId,
input: { question: "updated batch question" },
expectedOutput: null,
},
],
},
context,
),
).resolves.toMatchObject({
data: [
{
id: firstBatchItemId,
input: { question: "updated batch question" },
expectedOutput: null,
},
],
});
await expect(
handleBatchUpsertDatasetItems(
{ datasetId: dataset.id, items: [] },
context,
),
).rejects.toMatchObject({ name: "ZodError" });
await expect(
handleBatchUpsertDatasetItems(
{
datasetId: dataset.id,
items: Array.from({ length: 101 }, () => ({ input: {} })),
},
context,
),
).rejects.toMatchObject({ name: "ZodError" });
const datasetItems = (await handleListDatasetItems(
{ datasetId: dataset.id, page: 1, limit: 10 },
context,
)) as { data: Array<{ id: string }> };
expect(datasetItems.data.map((item) => item.id)).toContain(datasetItem.id);
await expect(
handleGetDatasetItem({ datasetItemId: datasetItem.id }, context),
).resolves.toMatchObject({
id: datasetItem.id,
datasetName: renamedDatasetName,
});
const runName = `mcp-run-50% accuracy %20 ${uuidv4()}`;
const runItem = (await handleCreateDatasetRunItem(
{
datasetItemId: datasetItem.id,
traceId,
observationId,
runName,
runDescription: "MCP run",
metadata: { source: "mcp" },
},
context,
)) as { id: string; datasetRunId: string; datasetItemId: string };
expect(runItem.datasetItemId).toBe(datasetItem.id);
await expect(
verifyAuditLog({
projectId,
apiKeyId,
resourceType: "datasetRunItem",
resourceId: runItem.id,
action: "create",
}),
).resolves.toMatchObject({
resourceType: "datasetRunItem",
resourceId: runItem.id,
action: "create",
});
await createDatasetRunItemsCh([
createDatasetRunItem({
id: runItem.id,
project_id: projectId,
trace_id: traceId,
observation_id: observationId,
dataset_run_id: runItem.datasetRunId,
dataset_item_id: datasetItem.id,
dataset_id: dataset.id,
dataset_run_name: runName,
}),
]);
await waitFor(async () => {
const runItems = (await handleListDatasetRunItems(
{
datasetId: dataset.id,
datasetRunId: runItem.datasetRunId,
page: 1,
limit: 10,
},
context,
)) as { data: Array<{ id: string }> };
return runItems.data.some((item) => item.id === runItem.id);
});
const datasetRuns = (await handleListDatasetRuns(
{ datasetId: dataset.id, page: 1, limit: 10 },
context,
)) as { data: Array<{ id: string; name: string }> };
expect(datasetRuns.data).toEqual(
expect.arrayContaining([
expect.objectContaining({ id: runItem.datasetRunId, name: runName }),
]),
);
await expect(
handleGetDatasetRun(
{ datasetId: dataset.id, datasetRunId: runItem.datasetRunId },
context,
),
).resolves.toMatchObject({
id: runItem.datasetRunId,
name: runName,
datasetRunItems: expect.arrayContaining([
expect.objectContaining({ id: runItem.id }),
]),
});
await expect(
handleDeleteDatasetRun(
{ datasetId: dataset.id, datasetRunId: runItem.datasetRunId },
context,
),
).resolves.toEqual({ message: "Dataset run successfully deleted" });
await expect(
handleDeleteDatasetItem({ datasetItemId: datasetItem.id }, context),
).resolves.toEqual({ message: "Dataset item successfully deleted" });
});
it("covers health public API route and cross-project recent-event checks", async () => {
const { context } = await createMcpTestSetup();
await expect(
handleGetHealth(
{} as unknown as Parameters<typeof handleGetHealth>[0],
context,
),
).resolves.toMatchObject({
status: "OK",
version: expect.any(String),
});
const { projectId: otherProjectId } = await createMcpTestSetup();
const traceId = uuidv4();
await createTracesCh([
createTrace({
id: traceId,
project_id: otherProjectId,
}),
]);
await createObservationsCh([
createObservation({
id: uuidv4(),
trace_id: traceId,
project_id: otherProjectId,
}),
]);
await expect(
handleGetHealth(
{ failIfNoRecentEvents: true } as unknown as Parameters<
typeof handleGetHealth
>[0],
context,
),
).resolves.toMatchObject({
status: "OK",
version: expect.any(String),
});
});
it("rejects cross-project access to project-scoped MCP resources", async () => {
const { context: sourceContext, projectId: sourceProjectId } =
await createMcpTestSetup();
const { context: targetContext } = await createMcpTestSetup();
const scoreConfig = await createScoreConfig(sourceProjectId);
const queue = (await handleCreateAnnotationQueue(
{
name: `mcp-queue-isolation-${uuidv4()}`,
scoreConfigIds: [scoreConfig.id],
},
sourceContext,
)) as { id: string };
await expect(
handleGetAnnotationQueue({ queueId: queue.id }, targetContext),
).rejects.toThrow("Annotation queue not found");
const datasetName = `mcp-dataset-isolation-${uuidv4()}`;
const dataset = (await handleUpsertDataset(
{ name: datasetName },
sourceContext,
)) as { id: string };
await expect(
handleGetDataset({ datasetId: dataset.id }, targetContext),
).rejects.toThrow("Dataset not found");
await expect(
handleListDatasetItems(
{ datasetId: dataset.id, page: 1, limit: 10 },
targetContext,
),
).rejects.toThrow("Dataset not found");
await expect(
handleListDatasetRunItems(
{ datasetId: dataset.id, datasetRunId: uuidv4(), page: 1, limit: 10 },
targetContext,
),
).rejects.toThrow("Dataset run not found");
const prompt = await createPromptInDb({
projectId: sourceProjectId,
name: `mcp-comment-isolation-${uuidv4()}`,
prompt: "Test prompt",
});
const comment = (await handleCreateComment(
{
objectType: "PROMPT",
objectId: prompt.id,
content: "MCP isolated comment",
},
sourceContext,
)) as { id: string };
await expect(
handleGetComment({ commentId: comment.id }, targetContext),
).rejects.toThrow("Comment not found within authorized project");
const modelName = `mcp-model-isolation-${uuidv4()}`;
const model = (await handleCreateModel(
{
modelName,
matchPattern: `(?i)^${modelName}$`,
unit: "TOKENS",
inputPrice: 0.001,
outputPrice: 0.002,
},
sourceContext,
)) as { id: string };
await expect(
handleGetModel({ modelId: model.id }, targetContext),
).rejects.toThrow("No model with this id found");
await expect(
handleGetScoreConfig({ configId: scoreConfig.id }, targetContext),
).rejects.toThrow("Score config not found within authorized project");
await expect(
handleUpdateScoreConfig(
{ configId: scoreConfig.id, description: "cross-project update" },
targetContext,
),
).rejects.toThrow("Score config not found within authorized project");
});
it("covers score config public API routes", async () => {
const { context } = await createMcpTestSetup();
const scoreConfigName = `mcp-score-${uuidv4().slice(0, 8)}`;
const scoreConfig = (await handleCreateScoreConfig(
{
name: scoreConfigName,
dataType: "NUMERIC",
numericMinValue: 0,
numericMaxValue: 1,
} as unknown as Parameters<typeof handleCreateScoreConfig>[0],
context,
)) as { id: string; name: string };
expect(scoreConfig.name).toBe(scoreConfigName);
const scoreConfigs = (await handleListScoreConfigs(
{ page: 1, limit: 10 },
context,
)) as { data: Array<{ id: string }> };
expect(scoreConfigs.data.map((item) => item.id)).toContain(scoreConfig.id);
await expect(
handleGetScoreConfig({ configId: scoreConfig.id }, context),
).resolves.toMatchObject({ id: scoreConfig.id, name: scoreConfigName });
await expect(
handleUpdateScoreConfig({ configId: scoreConfig.id }, context),
).rejects.toThrow("Request body cannot be empty");
await expect(
handleUpdateScoreConfig(
{
configId: scoreConfig.id,
description: "Updated through MCP",
},
context,
),
).resolves.toMatchObject({
id: scoreConfig.id,
description: "Updated through MCP",
});
});
});
describe("MCP tool schema interoperability", () => {
/** Recursively collect all JSON Schema `pattern` values with their location. */
const collectPatterns = (
schema: unknown,
path: string[] = [],
): { path: string; pattern: string }[] => {
if (typeof schema !== "object" || schema === null) return [];
if (Array.isArray(schema)) {
return schema.flatMap((item, index) =>
collectPatterns(item, [...path, String(index)]),
);
}
const obj = schema as Record<string, unknown>;
const ownPattern =
typeof obj.pattern === "string"
? [{ path: path.join("."), pattern: obj.pattern }]
: [];
return [
...ownPattern,
...Object.entries(obj).flatMap(([key, value]) =>
collectPatterns(value, [...path, key]),
),
];
};
const getAllToolPatterns = () =>
toolRegistry.getFeatures().flatMap((feature) =>
feature.tools.flatMap((tool) =>
collectPatterns(tool.definition.inputSchema).map((entry) => ({
tool: tool.definition.name,
...entry,
})),
),
);
it("advertises no pattern that requires the ECMAScript `u` flag", () => {
// JSON Schema `pattern` is an ECMA-262 regex compiled WITHOUT the `u` flag.
// Unicode-only escapes (`\p{...}`, `\P{...}`, `\u{...}`) don't merely fail
// OpenAI/Vertex validation — without `u` they silently degrade instead of
// throwing: e.g. `^[\p{L}\p{N}_ .()-]+$` collapses to a literal class
// `[pLN{}...]` that REJECTS valid ASCII like "quality". Because those
// providers validate the whole tool catalog atomically, a single such
// pattern disables every Langfuse MCP tool.
const offending = getAllToolPatterns().filter(({ pattern }) =>
/\\[pP]\{|\\u\{/.test(pattern),
);
expect(offending).toEqual([]);
});
});