🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
vi.hoisted(() => {
process.env.LANGFUSE_MIGRATION_V4_WRITE_MODE = "dual";
});
// Mock queue operations to avoid Redis dependency in tests
vi.mock("@langfuse/shared/src/server", async () => {
const actual = await vi.importActual("@langfuse/shared/src/server");
return {
...actual,
// Mock queue getInstance to return a no-op queue
EventPropagationQueue: {
getInstance: () => ({
add: vi.fn().mockResolvedValue(undefined),
disconnect: vi.fn(),
}),
},
};
});
vi.mock("@/src/features/media/server/getMediaStorageClient", () => ({
getMediaStorageServiceClient: () => ({
getSignedUrl: vi.fn().mockResolvedValue("https://media.example/download"),
}),
}));
// Skip evaluator configuration validation so these tool tests do not require
// a provisioned default eval model.
vi.mock(
"@/src/features/evals/server/evaluator-preflight",
async (importActual) => ({
...(await importActual<object>()),
getEvaluatorDefinitionConfigurationError: vi.fn(async () => null),
getEvaluatorDefinitionPreflightError: vi.fn(async () => null),
}),
);
import { nanoid } from "nanoid";
import { createHash, randomUUID } from "crypto";
import { z } from "zod";
import { prisma } from "@langfuse/shared/src/db";
import {
createDatasetRunScore,
createEvent,
createEventsCh,
createScoresCh,
createSessionScore,
createTraceScore,
buildDashboardUrl,
buildDashboardWidgetUrl,
buildEvaluatorUrl,
buildMonitorUrl,
buildExperimentUrl,
buildObservationUrl,
buildPromptUrl,
buildSessionUrl,
buildTraceUrl,
} from "@langfuse/shared/src/server";
import {
LLMAdapter,
ScoreConfigDataType,
supportedModels,
} from "@langfuse/shared";
import { encrypt } from "@langfuse/shared/encryption";
import { viewDeclarations } from "@langfuse/shared/query";
import { MonitorService } from "@langfuse/shared/monitors/server";
import {
createMcpTestSetup,
createPromptInDb,
mcpEvalOutputDefinition,
mockServerContext,
verifyAuditLog,
verifyToolAnnotations,
} from "./mcp-helpers";
import { env } from "@/src/env.mjs";
import "@/src/features/mcp/server/bootstrap";
import { toolRegistry } from "@/src/features/mcp/server/registry";
import { handleCreateDashboardWidget } from "@/src/features/mcp/server/dashboardWidgets/tools/createDashboardWidget";
import {
getDashboardTool,
getDashboardWidgetTool,
handleCreateDashboard,
handleGetDashboard,
handleGetDashboardWidget,
handleListDashboards,
handleListDashboardWidgets,
listDashboardsTool,
listDashboardWidgetsTool,
} from "@/src/features/mcp/server/dashboardWidgets/tools/dashboardCrud";
// Import MCP tool handlers directly
import {
getObservationTool,
handleGetObservation,
} from "@/src/features/mcp/server/observations/tools/getObservation";
import {
getObservationFieldSchemaTool,
handleGetObservationFieldSchema,
} from "@/src/features/mcp/server/observations/tools/getObservationFieldSchema";
import {
getObservationFilterSchemaTool,
handleGetObservationFilterSchema,
} from "@/src/features/mcp/server/observations/tools/getObservationFilterSchema";
import {
getObservationFilterValuesTool,
handleGetObservationFilterValues,
} from "@/src/features/mcp/server/observations/tools/getObservationFilterValues";
import {
listObservationsTool,
handleListObservations,
} from "@/src/features/mcp/server/observations/tools/listObservations";
import {
getMetricsSchemaTool,
handleGetMetricsSchema,
} from "@/src/features/mcp/server/metrics/tools/getMetricsSchema";
import {
queryMetricsTool,
handleQueryMetrics,
} from "@/src/features/mcp/server/metrics/tools/queryMetrics";
import {
getPromptTool,
handleGetPrompt,
} from "@/src/features/mcp/server/prompts/tools/getPrompt";
import {
getPromptUnresolvedTool,
handleGetPromptUnresolved,
} from "@/src/features/mcp/server/prompts/tools/getPromptUnresolved";
import {
listPromptsTool,
handleListPrompts,
} from "@/src/features/mcp/server/prompts/tools/listPrompts";
import {
createScoreConfigTool,
handleCreateScoreConfig,
} from "@/src/features/mcp/server/scores/tools/createScoreConfig";
import {
createScoreTool,
handleCreateScore,
} from "@/src/features/mcp/server/scores/tools/createScore";
import {
deleteScoreConfigTool,
handleDeleteScoreConfig,
} from "@/src/features/mcp/server/scores/tools/deleteScoreConfig";
import {
getScoreTool,
handleGetScore,
} from "@/src/features/mcp/server/scores/tools/getScore";
import {
getScoreConfigTool,
handleGetScoreConfig,
} from "@/src/features/mcp/server/scores/tools/getScoreConfig";
import {
listScoreConfigsTool,
handleListScoreConfigs,
} from "@/src/features/mcp/server/scores/tools/listScoreConfigs";
import {
listScoresTool,
handleListScores,
} from "@/src/features/mcp/server/scores/tools/listScores";
import {
updateScoreConfigTool,
handleUpdateScoreConfig,
} from "@/src/features/mcp/server/scores/tools/updateScoreConfig";
import {
getMediaTool,
handleGetMedia,
} from "@/src/features/mcp/server/media/tools/getMedia";
import {
getEvaluatorTool,
handleGetEvaluator,
} from "@/src/features/mcp/server/evals/tools/getEvaluator";
import {
listEvaluatorsTool,
handleListEvaluators,
} from "@/src/features/mcp/server/evals/tools/listEvaluators";
import {
handleListManagedEvaluatorTemplates,
listManagedEvaluatorTemplatesTool,
} from "@/src/features/mcp/server/evals/tools/listManagedEvaluatorTemplates";
import {
getEvaluationRuleTool,
handleGetEvaluationRule,
} from "@/src/features/mcp/server/evals/tools/getEvaluationRule";
import {
listEvaluationRulesTool,
handleListEvaluationRules,
} from "@/src/features/mcp/server/evals/tools/listEvaluationRules";
import { handleCreateEvaluator } from "@/src/features/mcp/server/evals/tools/createEvaluator";
import { handleCreateEvaluationRule } from "@/src/features/mcp/server/evals/tools/createEvaluationRule";
import {
getMonitorTool,
handleGetMonitor,
} from "@/src/features/mcp/server/monitors/tools/getMonitor";
import {
listMonitorsTool,
handleListMonitors,
} from "@/src/features/mcp/server/monitors/tools/listMonitors";
import {
GetDatasetItemsMcpInput,
GetDatasetMcpInput,
GetDatasetRunMcpInput,
GetDatasetRunsMcpInput,
GetDatasetsMcpInput,
} from "@/src/features/mcp/server/datasets/schema";
import {
handleListLlmConnections,
listLlmConnectionsTool,
} from "@/src/features/mcp/server/llmConnections/tools/listLlmConnections";
import { SkillService } from "@/src/features/skills/server/skill-service";
import { handleListSkills } from "@/src/features/mcp/server/skills/tools/listSkills";
import { handleGetSkill } from "@/src/features/mcp/server/skills/tools/getSkill";
import { handleLoadSkill } from "@/src/features/mcp/server/skills/tools/loadSkill";
import { handleLoadSkillResource } from "@/src/features/mcp/server/skills/tools/loadSkillResource";
const observationWindow = {
fromStartTime: new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString(),
toStartTime: new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString(),
};
const maybeEventsTable =
env.LANGFUSE_MIGRATION_V4_ALLOW_PREVIEW_OPT_IN === "true"
? describe
: describe.skip;
const maybeEventsTableIt =
env.LANGFUSE_MIGRATION_V4_ALLOW_PREVIEW_OPT_IN === "true" ? it : it.skip;
const createLlmEvaluatorForMcpReadTest = async (
setup: Awaited<ReturnType<typeof createMcpTestSetup>>,
name = `mcp-eval-${nanoid()}`,
) => {
return (await handleCreateEvaluator(
{
name,
type: "LLM_AS_JUDGE",
prompt: "Judge {{input}} against {{output}}",
outputDefinition: mcpEvalOutputDefinition,
},
setup.context,
)) as { id: string; name: string };
};
const createMonitorForMcpReadTest = async (
setup: Awaited<ReturnType<typeof createMcpTestSetup>>,
params: { name: string; tags?: string[] },
) => {
const user = await prisma.user.create({
data: {
id: randomUUID(),
email: `mcp-monitor-${nanoid()}@example.com`,
name: "MCP Monitor Test User",
},
});
return MonitorService.create(
{ userId: user.id },
{
projectId: setup.projectId,
view: "observations",
filters: [],
metric: { measure: "count", aggregation: "count" },
window: "5m",
thresholdOperator: "GT",
alertThreshold: 100,
warningThreshold: null,
noData: { mode: "SHOW_NO_DATA" },
renotify: { mode: "OFF" },
status: "ACTIVE",
name: params.name,
tags: params.tags ?? [],
triggerIds: ["mcp-test-trigger"],
},
);
};
const createEvaluationRuleForMcpReadTest = async (
setup: Awaited<ReturnType<typeof createMcpTestSetup>>,
) => {
const evaluatorName = `mcp-eval-${nanoid()}`;
const evaluator = await createLlmEvaluatorForMcpReadTest(
setup,
evaluatorName,
);
const ruleName = `mcp-rule-${nanoid()}`;
const rule = (await handleCreateEvaluationRule(
{
name: ruleName,
evaluatorAssignments: [
{
evaluatorId: evaluator.id,
variableMapping: [
{ variable: "input", source: "input" },
{ variable: "output", source: "output" },
],
},
],
enabled: true,
sampling: 1,
filter: [
{ column: "version", operator: "=", value: "1.0.0", type: "string" },
],
},
setup.context,
)) as { id: string; name: string };
return { evaluator, rule };
};
const createObservationEvent = (params: {
projectId: string;
traceId?: string;
observationId?: string;
name?: string;
type?: "GENERATION" | "SPAN" | "EVENT";
startTime?: Date;
parentObservationId?: string | null;
isAppRoot?: boolean;
providedModelName?: string;
input?: string;
output?: string;
metadata?: Record<string, string>;
tags?: string[];
userId?: string;
sessionId?: string;
totalCost?: number;
}) => {
const observationId = params.observationId ?? randomUUID();
const startTime = params.startTime ?? new Date();
const metadata = params.metadata ?? { source: "mcp-test" };
const metadataKeys = Object.keys(metadata).sort();
return createEvent({
id: observationId,
span_id: observationId,
trace_id: params.traceId ?? randomUUID(),
parent_span_id: params.parentObservationId ?? null,
is_app_root: params.isAppRoot ?? false,
project_id: params.projectId,
name: params.name ?? `mcp-observation-${nanoid()}`,
type: params.type ?? "GENERATION",
level: "DEFAULT",
start_time: startTime.getTime() * 1000,
end_time: startTime.getTime() * 1000 + 1000 * 1000,
input: params.input ?? "Observation input",
output: params.output ?? "Observation output",
metadata_names: metadataKeys,
metadata_values: metadataKeys.map((key) => metadata[key]),
provided_model_name: params.providedModelName ?? "gpt-4o-mini",
...(params.totalCost === undefined
? {}
: { cost_details: { total: params.totalCost } }),
tags: params.tags ?? [],
user_id: params.userId ?? null,
session_id: params.sessionId ?? null,
});
};
const isRecord = (value: unknown): value is Record<string, unknown> =>
Boolean(value) && typeof value === "object" && !Array.isArray(value);
const containsStringValue = (value: unknown, expected: string): boolean => {
if (typeof value === "string") {
return value.includes(expected);
}
if (Array.isArray(value)) {
return value.some((item) => containsStringValue(item, expected));
}
if (isRecord(value)) {
return Object.values(value).some((item) =>
containsStringValue(item, expected),
);
}
return false;
};
const containsPropertyName = (value: unknown, expected: string): boolean => {
if (Array.isArray(value)) {
return value.some((item) => containsPropertyName(item, expected));
}
if (isRecord(value)) {
return (
Object.keys(value).includes(expected) ||
Object.values(value).some((item) => containsPropertyName(item, expected))
);
}
return false;
};
describe("MCP Read Tools", () => {
describe("skills tools", () => {
const createSkillForMcpReadTest = async () => {
const setup = await createMcpTestSetup();
const service = new SkillService(prisma);
const name = `mcp-skill-${randomUUID()}`;
const root = `---\nname: ${name}\ndescription: Review customer requests\n---\nFollow the reference checklist.\n`;
const reference = "Review the customer's résumé before proceeding.\n";
const actor = {
projectId: setup.projectId,
orgId: setup.orgId,
apiKeyId: setup.apiKeyId,
accessLevel: "project" as const,
};
const version = await service.createVersion({
projectId: setup.projectId,
createdBy: "API",
input: {
files: [
{ path: "SKILL.md", content: root },
{ path: "references/checklist.md", content: reference },
],
},
actor,
});
return { ...setup, service, actor, name, root, reference, version };
};
it("lists skill metadata and loads the resolved version's manifest and raw text", async () => {
const fixture = await createSkillForMcpReadTest();
const manifest = (await handleGetSkill(
{ name: fixture.name, label: "latest" },
fixture.context,
)) as Awaited<ReturnType<SkillService["get"]>>;
expect(manifest).toMatchObject({
name: fixture.name,
version: fixture.version.version,
files: expect.arrayContaining([
expect.objectContaining({ path: "SKILL.md" }),
expect.objectContaining({ path: "references/checklist.md" }),
]),
});
expect(manifest.files.every((file) => !("content" in file))).toBe(true);
await fixture.service.createVersion({
projectId: fixture.projectId,
createdBy: "API",
input: {
files: [
{
path: "SKILL.md",
content: `${fixture.root}\nUpdated instructions.`,
},
{ path: "references/checklist.md", content: "Updated checklist." },
],
},
actor: fixture.actor,
});
await expect(
handleListSkills(
{ name: fixture.name, page: 1, limit: 10 },
fixture.context,
),
).resolves.toMatchObject({
data: [
{ name: fixture.name, latestVersion: 2, productionVersion: null },
],
meta: { totalItems: 1 },
});
const selector = { name: fixture.name, version: manifest.version };
await expect(handleLoadSkill(selector, fixture.context)).resolves.toBe(
fixture.root,
);
await expect(
handleLoadSkillResource(
{ ...selector, path: "references/checklist.md" },
fixture.context,
),
).resolves.toBe(fixture.reference);
});
it("defaults to production, then latest, when no skill selector is provided", async () => {
const fixture = await createSkillForMcpReadTest();
const latestRoot = `${fixture.root}\nUpdated instructions.`;
const latestReference = "Updated checklist.";
await fixture.service.createVersion({
projectId: fixture.projectId,
createdBy: "API",
input: {
files: [
{ path: "SKILL.md", content: latestRoot },
{ path: "references/checklist.md", content: latestReference },
{ path: "latest-only.txt", content: "Only in latest." },
],
},
actor: fixture.actor,
});
const expectDefaultVersion = async (
version: number,
root: string,
reference: string,
) => {
await expect(
handleGetSkill({ name: fixture.name }, fixture.context),
).resolves.toMatchObject({ version });
await expect(
handleLoadSkill({ name: fixture.name }, fixture.context),
).resolves.toBe(root);
await expect(
handleLoadSkillResource(
{ name: fixture.name, path: "references/checklist.md" },
fixture.context,
),
).resolves.toBe(reference);
};
await expectDefaultVersion(2, latestRoot, latestReference);
await fixture.service.setLabels({
projectId: fixture.projectId,
name: fixture.name,
version: 1,
labels: ["production"],
actor: fixture.actor,
});
await expectDefaultVersion(1, fixture.root, fixture.reference);
await expect(
handleLoadSkillResource(
{ name: fixture.name, path: "latest-only.txt" },
fixture.context,
),
).rejects.toThrow("Skill resource not found");
await expect(
handleGetSkill({ name: fixture.name, version: 2 }, fixture.context),
).resolves.toMatchObject({ version: 2 });
await expect(
handleLoadSkill(
{ name: fixture.name, label: "latest" },
fixture.context,
),
).resolves.toBe(latestRoot);
});
it.each([
["getSkill", handleGetSkill],
["loadSkill", handleLoadSkill],
["loadSkillResource", handleLoadSkillResource],
] as const)(
"does not fall back for missing explicit selectors in %s",
async (_name, handler) => {
const fixture = await createSkillForMcpReadTest();
for (const selector of [{ version: 99 }, { label: "production" }]) {
await expect(
handler(
{ name: fixture.name, path: "SKILL.md", ...selector },
fixture.context,
),
).rejects.toThrow("Skill version not found");
}
await expect(
handler(
{ name: `missing-${randomUUID()}`, path: "SKILL.md" },
fixture.context,
),
).rejects.toThrow("Skill version not found");
},
);
it.each([
["getSkill", handleGetSkill],
["loadSkill", handleLoadSkill],
["loadSkillResource", handleLoadSkillResource],
] as const)(
"rejects conflicting selectors in %s",
async (_name, handler) => {
await expect(
handler(
{
name: "test-skill",
version: 1,
label: "latest",
path: "SKILL.md",
},
mockServerContext(),
),
).rejects.toThrow("Specify either version or label, not both");
},
);
it("keeps skill metadata and contents isolated to the authenticated project", async () => {
const fixture = await createSkillForMcpReadTest();
const other = await createMcpTestSetup();
const selector = { name: fixture.name, version: fixture.version.version };
await expect(
handleListSkills(
{ name: fixture.name, page: 1, limit: 10 },
other.context,
),
).resolves.toMatchObject({ data: [], meta: { totalItems: 0 } });
for (const handler of [
handleGetSkill,
handleLoadSkill,
handleLoadSkillResource,
]) {
for (const input of [selector, { name: fixture.name }]) {
await expect(
handler(
{ ...input, path: "references/checklist.md" },
other.context,
),
).rejects.toThrow("Skill version not found");
}
}
});
});
describe("listMonitors tool", () => {
it("should have readOnlyHint annotation", () => {
verifyToolAnnotations(listMonitorsTool, { readOnlyHint: true });
});
it("should be available to in-app agent keys", async () => {
const context = mockServerContext({
inAppAgent: { permissions: "read" },
});
await expect(
toolRegistry.getEnabledTool(listMonitorsTool.name, context),
).resolves.toBeDefined();
});
it("should not expose projectId in its input schema", () => {
const properties = listMonitorsTool.inputSchema.properties as Record<
string,
unknown
>;
expect(properties.projectId).toBeUndefined();
});
it("should list, filter, order, and paginate monitors in the current project", async () => {
const setup = await createMcpTestSetup();
const otherSetup = await createMcpTestSetup();
const tag = `mcp-monitor-${nanoid()}`;
const first = await createMonitorForMcpReadTest(setup, {
name: `A monitor ${nanoid()}`,
tags: [tag],
});
await createMonitorForMcpReadTest(setup, {
name: `B monitor ${nanoid()}`,
tags: [tag],
});
await createMonitorForMcpReadTest(otherSetup, {
name: `Other project monitor ${nanoid()}`,
tags: [tag],
});
const result = (await handleListMonitors(
{
orderBy: { column: "name", order: "ASC" },
filter: [
{
type: "arrayOptions",
column: "tags",
operator: "any of",
value: [tag],
},
],
page: 1,
limit: 1,
},
setup.context,
)) as {
data: Array<{ id: string; url: string }>;
meta: { totalItems: number; totalPages: number };
};
expect(result).toEqual({
data: [
expect.objectContaining({
id: first.id,
url: buildMonitorUrl({
projectId: setup.projectId,
monitorId: first.id,
}),
}),
],
meta: expect.objectContaining({ totalItems: 2, totalPages: 2 }),
});
});
});
describe("getMonitor tool", () => {
it("should have readOnlyHint annotation", () => {
verifyToolAnnotations(getMonitorTool, { readOnlyHint: true });
});
it("should be available to in-app agent keys", async () => {
const context = mockServerContext({
inAppAgent: { permissions: "read" },
});
await expect(
toolRegistry.getEnabledTool(getMonitorTool.name, context),
).resolves.toBeDefined();
});
it("should fetch a monitor by id", async () => {
const setup = await createMcpTestSetup();
const monitor = await createMonitorForMcpReadTest(setup, {
name: `Get monitor ${nanoid()}`,
});
await expect(
handleGetMonitor({ monitorId: monitor.id }, setup.context),
).resolves.toMatchObject({
id: monitor.id,
name: monitor.name,
url: buildMonitorUrl({
projectId: setup.projectId,
monitorId: monitor.id,
}),
});
});
it("should reject missing and cross-project monitors", async () => {
const setup = await createMcpTestSetup();
const otherSetup = await createMcpTestSetup();
const monitor = await createMonitorForMcpReadTest(setup, {
name: `Isolated monitor ${nanoid()}`,
});
await expect(
handleGetMonitor({ monitorId: randomUUID() }, setup.context),
).rejects.toThrow(/not found/i);
await expect(
handleGetMonitor({ monitorId: monitor.id }, otherSetup.context),
).rejects.toThrow(/not found/i);
});
});
describe("listLlmConnections tool", () => {
it("should have readOnlyHint annotation", () => {
verifyToolAnnotations(listLlmConnectionsTool, { readOnlyHint: true });
});
it("should be available to in-app agent keys", async () => {
const context = mockServerContext({
inAppAgent: { permissions: "read" },
});
await expect(
toolRegistry.getEnabledTool(listLlmConnectionsTool.name, context),
).resolves.toBeDefined();
});
it("should list effective project and organization connections without credentials", async () => {
const setup = await createMcpTestSetup();
const otherSetup = await createMcpTestSetup();
const secret = `sk-mcp-secret-${nanoid()}`;
const sharedProvider = `shared-${nanoid()}`;
const orgOnlyProvider = `org-only-${nanoid()}`;
const otherOrgProvider = `other-org-${nanoid()}`;
const connection = (
owner: { projectId: string } | { organizationId: string },
data: {
provider: string;
adapter: LLMAdapter;
customModels?: string[];
withDefaultModels?: boolean;
},
) =>
prisma.llmApiKeys.create({
data: {
...owner,
...data,
secretKey: encrypt(secret),
displaySecretKey: "...cret",
extraHeaders: encrypt(JSON.stringify({ "x-secret": secret })),
},
});
await connection(
{ projectId: setup.projectId },
{
provider: sharedProvider,
adapter: LLMAdapter.OpenAI,
customModels: ["my-model", "gpt-4o"],
withDefaultModels: true,
},
);
await connection(
{ organizationId: setup.orgId },
{ provider: sharedProvider, adapter: LLMAdapter.Anthropic },
);
await connection(
{ organizationId: setup.orgId },
{
provider: orgOnlyProvider,
adapter: LLMAdapter.Anthropic,
customModels: ["claude-custom"],
withDefaultModels: false,
},
);
await connection(
{ organizationId: otherSetup.orgId },
{ provider: otherOrgProvider, adapter: LLMAdapter.OpenAI },
);
const result = (await handleListLlmConnections({}, setup.context)) as {
data: Array<Record<string, unknown>>;
};
expect(result.data.map((c) => c.provider).sort()).toEqual(
[orgOnlyProvider, sharedProvider].sort(),
);
expect(result.data).toEqual(
expect.arrayContaining([
expect.objectContaining({
provider: sharedProvider,
adapter: LLMAdapter.OpenAI,
scope: "project",
displaySecretKey: "...cret",
availableModels: Array.from(
new Set([
"my-model",
"gpt-4o",
...supportedModels[LLMAdapter.OpenAI],
]),
),
}),
expect.objectContaining({
provider: orgOnlyProvider,
adapter: LLMAdapter.Anthropic,
scope: "organization",
availableModels: ["claude-custom"],
}),
]),
);
for (const item of result.data) {
expect(item).not.toHaveProperty("secretKey");
expect(item).not.toHaveProperty("extraHeaders");
expect(item).not.toHaveProperty("projectId");
expect(item).not.toHaveProperty("organizationId");
}
expect(JSON.stringify(result)).not.toContain(secret);
});
});
describe("dataset tool schemas", () => {
it("uses dataset IDs for existing dataset read addressing", () => {
for (const schema of [
GetDatasetMcpInput,
GetDatasetItemsMcpInput,
GetDatasetRunsMcpInput,
GetDatasetRunMcpInput,
]) {
const jsonSchema = z.toJSONSchema(schema, { unrepresentable: "any" });
const properties = jsonSchema.properties as Record<string, unknown>;
expect(properties).toHaveProperty("datasetId");
expect(properties).not.toHaveProperty("datasetName");
expect(properties).not.toHaveProperty("name");
}
});
it("keeps dataset names only as a listDatasets discovery filter", () => {
const jsonSchema = z.toJSONSchema(GetDatasetsMcpInput, {
unrepresentable: "any",
});
const properties = jsonSchema.properties as Record<string, unknown>;
expect(properties).toHaveProperty("name");
expect(properties).not.toHaveProperty("datasetId");
});
});
describe("listEvaluators tool", () => {
it("should have readOnlyHint annotation", () => {
verifyToolAnnotations(listEvaluatorsTool, { readOnlyHint: true });
});
it("should be available to in-app agent keys", async () => {
const context = mockServerContext({
inAppAgent: { permissions: "read" },
});
await expect(
toolRegistry.getEnabledTool(listEvaluatorsTool.name, context),
).resolves.toMatchObject({
definition: expect.objectContaining({ name: listEvaluatorsTool.name }),
});
});
it("should list evaluators for the current project", async () => {
const setup = await createMcpTestSetup();
const { context } = setup;
const evaluator = await createLlmEvaluatorForMcpReadTest(setup);
const result = (await handleListEvaluators(
{ page: 1, limit: 50 },
context,
)) as { data: Array<{ id: string; url: string }> };
expect(result.data.map((item) => item.id)).toContain(evaluator.id);
expect(result.data.find((item) => item.id === evaluator.id)?.url).toBe(
buildEvaluatorUrl({
projectId: setup.projectId,
evaluatorId: evaluator.id,
}),
);
});
});
describe("listManagedEvaluatorTemplates tool", () => {
it("lists filtered managed templates without database state", async () => {
verifyToolAnnotations(listManagedEvaluatorTemplatesTool, {
readOnlyHint: true,
});
const result = (await handleListManagedEvaluatorTemplates(
{ type: "CODE" },
mockServerContext(),
)) as {
schemaVersion: number;
templates: Array<{ key: string; evaluator: { type: string } }>;
};
expect(result.schemaVersion).toBe(1);
expect(result.templates).toEqual(
expect.arrayContaining([
expect.objectContaining({
key: "exact-match",
evaluator: expect.objectContaining({ type: "CODE" }),
}),
]),
);
expect(result.templates.length).toBeGreaterThan(0);
expect(
result.templates.every(
(template) => template.evaluator.type === "CODE",
),
).toBe(true);
});
it("lists decision-model templates", async () => {
const result = (await handleListManagedEvaluatorTemplates(
{ type: "DECISION_MODEL" },
mockServerContext(),
)) as {
templates: Array<{ key: string; evaluator: { type: string } }>;
};
expect(result.templates).toEqual(
expect.arrayContaining([
expect.objectContaining({
key: "topic-decision-model",
evaluator: expect.objectContaining({
type: "DECISION_MODEL",
}),
}),
]),
);
expect(
result.templates.every(
(template) => template.evaluator.type === "DECISION_MODEL",
),
).toBe(true);
});
});
describe("getEvaluator tool", () => {
it("should have readOnlyHint annotation", () => {
verifyToolAnnotations(getEvaluatorTool, { readOnlyHint: true });
});
it("should be available to in-app agent keys", async () => {
const context = mockServerContext({
inAppAgent: { permissions: "read" },
});
await expect(
toolRegistry.getEnabledTool(getEvaluatorTool.name, context),
).resolves.toMatchObject({
definition: expect.objectContaining({ name: getEvaluatorTool.name }),
});
});
it("should fetch an evaluator by id", async () => {
const setup = await createMcpTestSetup();
const evaluator = await createLlmEvaluatorForMcpReadTest(setup);
await expect(
handleGetEvaluator({ evaluatorId: evaluator.id }, setup.context),
).resolves.toMatchObject({
id: evaluator.id,
name: evaluator.name,
url: buildEvaluatorUrl({
projectId: setup.projectId,
evaluatorId: evaluator.id,
}),
});
});
});
describe("listEvaluationRules tool", () => {
it("should have readOnlyHint annotation", () => {
verifyToolAnnotations(listEvaluationRulesTool, { readOnlyHint: true });
});
it("should be available to in-app agent keys", async () => {
const context = mockServerContext({
inAppAgent: { permissions: "read" },
});
await expect(
toolRegistry.getEnabledTool(listEvaluationRulesTool.name, context),
).resolves.toMatchObject({
definition: expect.objectContaining({
name: listEvaluationRulesTool.name,
}),
});
});
it("should list evaluation rules for the current project", async () => {
const setup = await createMcpTestSetup();
const { rule } = await createEvaluationRuleForMcpReadTest(setup);
const result = (await handleListEvaluationRules(
{ page: 1, limit: 50 },
setup.context,
)) as { data: Array<{ id: string }> };
expect(result.data.map((item) => item.id)).toContain(rule.id);
});
it("should list incomplete evaluator mappings", async () => {
const setup = await createMcpTestSetup();
const evaluator = await createLlmEvaluatorForMcpReadTest(setup);
const rule = await prisma.evaluationRule.create({
data: {
projectId: setup.projectId,
name: `mcp-incomplete-mapping-rule-${nanoid()}`,
targetObject: "event",
filter: [],
sampling: 1,
delay: 0,
assignments: {
create: {
projectId: setup.projectId,
evaluatorId: evaluator.id,
variableMapping: [
{
templateVariable: "input",
selectedColumnId: "",
jsonSelector: null,
},
{
templateVariable: "output",
selectedColumnId: "output",
jsonSelector: null,
},
],
},
},
},
});
const result = (await handleListEvaluationRules(
{ page: 1, limit: 50 },
setup.context,
)) as {
data: Array<{
id: string;
evaluators: Array<{
evaluatorId: string;
variableMapping: Array<{
variable: string;
source: string | null;
}> | null;
}>;
}>;
};
expect(result.data.find((item) => item.id === rule.id)).toMatchObject({
evaluators: [
{
evaluatorId: evaluator.id,
variableMapping: [
{ variable: "input", source: null },
{ variable: "output", source: "output" },
],
},
],
});
});
it("returns multi-evaluator rules with all assignments", async () => {
const setup = await createMcpTestSetup();
const [firstEvaluator, secondEvaluator] = await Promise.all([
createLlmEvaluatorForMcpReadTest(setup),
createLlmEvaluatorForMcpReadTest(setup),
]);
const rule = await prisma.evaluationRule.create({
data: {
projectId: setup.projectId,
name: `mcp-multi-evaluator-rule-${nanoid()}`,
targetObject: "event",
filter: [],
sampling: 1,
delay: 0,
assignments: {
create: [firstEvaluator, secondEvaluator].map((evaluator) => ({
projectId: setup.projectId,
evaluatorId: evaluator.id,
variableMapping: [],
})),
},
},
});
const result = (await handleListEvaluationRules(
{ page: 1, limit: 50 },
setup.context,
)) as { data: Array<{ id: string }> };
expect(result.data).toContainEqual(
expect.objectContaining({
id: rule.id,
evaluators: expect.arrayContaining([
expect.objectContaining({ evaluatorId: firstEvaluator.id }),
expect.objectContaining({ evaluatorId: secondEvaluator.id }),
]),
}),
);
await expect(
handleGetEvaluationRule({ evaluationRuleId: rule.id }, setup.context),
).resolves.toMatchObject({
id: rule.id,
evaluators: expect.arrayContaining([
expect.objectContaining({ evaluatorId: firstEvaluator.id }),
expect.objectContaining({ evaluatorId: secondEvaluator.id }),
]),
});
});
});
describe("getEvaluationRule tool", () => {
it("should have readOnlyHint annotation", () => {
verifyToolAnnotations(getEvaluationRuleTool, { readOnlyHint: true });
});
it("should be available to in-app agent keys", async () => {
const context = mockServerContext({
inAppAgent: { permissions: "read" },
});
await expect(
toolRegistry.getEnabledTool(getEvaluationRuleTool.name, context),
).resolves.toMatchObject({
definition: expect.objectContaining({
name: getEvaluationRuleTool.name,
}),
});
});
it("should fetch an evaluation rule by id", async () => {
const setup = await createMcpTestSetup();
const { rule } = await createEvaluationRuleForMcpReadTest(setup);
await expect(
handleGetEvaluationRule({ evaluationRuleId: rule.id }, setup.context),
).resolves.toMatchObject({ id: rule.id, name: rule.name });
});
});
describe("getMedia tool", () => {
it("should have readOnlyHint annotation", () => {
verifyToolAnnotations(getMediaTool, { readOnlyHint: true });
});
it("should return media metadata and a signed download URL", async () => {
const { context, projectId } = await createMcpTestSetup();
const mediaId = `mcp-media-${nanoid()}`;
const uploadedAt = new Date("2026-01-01T00:00:00.000Z");
await prisma.media.create({
data: {
id: mediaId,
projectId,
sha256Hash: createHash("sha256").update(mediaId).digest("base64"),
bucketPath: `${projectId}/${mediaId}.png`,
bucketName: "media-test-bucket",
contentType: "image/png",
contentLength: 123n,
uploadedAt,
uploadHttpStatus: 200,
},
});
const result = (await handleGetMedia({ mediaId }, context)) as {
mediaId: string;
contentType: string;
contentLength: number;
uploadedAt: Date;
url: string;
urlExpiry: string;
};
expect(result).toMatchObject({
mediaId,
contentType: "image/png",
contentLength: 123,
url: "https://media.example/download",
});
expect(result.uploadedAt).toEqual(uploadedAt);
expect(new Date(result.urlExpiry).getTime()).toBeGreaterThan(Date.now());
});
});
describe("getObservationFieldSchema tool", () => {
it("should have readOnlyHint annotation", () => {
verifyToolAnnotations(getObservationFieldSchemaTool, {
readOnlyHint: true,
});
});
maybeEventsTableIt("should be available to in-app agent keys", async () => {
const context = mockServerContext({
inAppAgent: { permissions: "read" },
});
await expect(
toolRegistry.getEnabledTool(
getObservationFieldSchemaTool.name,
context,
),
).resolves.toBeDefined();
});
it("should return the observation projection field schema", async () => {
const result = (await handleGetObservationFieldSchema(
{},
mockServerContext(),
)) as {
resource: string;
defaultFields: string[];
fields: Record<
string,
{
type: string;
nullable: boolean;
default: boolean;
expensive: boolean;
sensitive: boolean;
requiresScope?: boolean;
scopeRequirement?: string;
description?: string;
}
>;
};
expect(result.resource).toBe("observation");
expect(result.defaultFields).toEqual(
expect.arrayContaining([
"id",
"traceId",
"parentObservationId",
"name",
"type",
"level",
"statusMessage",
"startTime",
"endTime",
"latency",
"providedModelName",
]),
);
expect(result.fields.providedModelName.default).toBe(true);
expect(result.fields.providedModelName.type).toBe("string");
expect(result.fields.providedModelName.nullable).toBe(true);
expect(result.fields.startTime.type).toBe("datetime");
expect(result.fields.startTime.nullable).toBe(false);
expect(result.fields.isRootObservation.type).toBe("boolean");
expect(result.fields.costDetails.type).toBe("map<string, number>");
expect(result.fields.input.expensive).toBe(true);
expect(result.fields.input.sensitive).toBe(true);
expect(result.fields.input.requiresScope).toBe(true);
expect(result.fields.input.scopeRequirement).toMatch(
/requires both fromStartTime.*toStartTime.*even when scoped by traceId or observation id.*listObservations.*14 days \(336 elapsed hours\).*getObservation.*90 days \(2160 elapsed hours\)/i,
);
expect(result.fields.output.requiresScope).toBe(true);
expect(result.fields.metadata.expensive).toBe(true);
expect(result.fields.metadata.requiresScope).toBe(true);
expect(result.fields.metadata.description).toContain(
"truncated to 200 UTF-8 characters per key",
);
expect(result.fields.userId.sensitive).toBe(true);
});
});
describe("getObservationFilterSchema tool", () => {
it("should have readOnlyHint annotation", () => {
verifyToolAnnotations(getObservationFilterSchemaTool, {
readOnlyHint: true,
});
});
maybeEventsTableIt("should be available to in-app agent keys", async () => {
const context = mockServerContext({
inAppAgent: { permissions: "read" },
});
await expect(
toolRegistry.getEnabledTool(
getObservationFilterSchemaTool.name,
context,
),
).resolves.toBeDefined();
});
it("should return public-compatible observation filter schema", async () => {
const result = (await handleGetObservationFilterSchema(
{},
mockServerContext(),
)) as {
resource: string;
columns: Record<string, { type: string; operators: string[] }>;
};
expect(result.resource).toBe("observation");
expect(result.columns.providedModelName.type).toBe("stringOptions");
expect(result.columns.tags.type).toBe("arrayOptions");
expect(result.columns.metadata).toEqual(
expect.objectContaining({
type: "stringObject",
operators: expect.arrayContaining(["matches"]),
requiresKey: true,
}),
);
expect(result.columns.input.operators).toEqual(["=", "matches"]);
expect(result.columns.output.operators).toEqual(["=", "matches"]);
expect(result.columns.version.operators).not.toContain("matches");
expect(result.columns.traceTags).toBeUndefined();
expect(result.columns.startTime).toBeUndefined();
expect(result.columns.comments).toBeUndefined();
expect(result.columns.scores).toBeUndefined();
expect(result.columns.name.operators).toContain("any of");
});
it.each([
["name", "stringOptions", true],
["type", "stringOptions", false],
["environment", "stringOptions", true],
["version", "string", true],
["userId", "string", true],
["sessionId", "string", true],
["traceName", "stringOptions", true],
["level", "stringOptions", false],
["promptName", "stringOptions", true],
["modelId", "stringOptions", true],
["providedModelName", "stringOptions", true],
["tags", "arrayOptions", false],
["isRootObservation", "boolean", false],
["hasParentObservation", "boolean", false],
])(
"should expose the %s column used by observation filter values",
async (column, type, nullable) => {
const { context } = await createMcpTestSetup();
const result = (await handleGetObservationFilterSchema(
{},
context,
)) as {
columns: Record<
string,
{ type: string; operators: string[]; nullable: boolean }
>;
};
expect(result.columns[column]).toEqual(
expect.objectContaining({
type,
nullable,
operators: expect.any(Array),
}),
);
expect(result.columns[column]?.operators.length).toBeGreaterThan(0);
},
);
});
it.each([
["input", "contains"],
["version", "matches"],
] as const)("should reject %s filters using %s", async (column, operator) => {
await expect(
handleListObservations(
{
traceId: randomUUID(),
...observationWindow,
filter: [
{
type: "string",
column,
operator,
value: "needle",
},
],
fields: ["id"],
limit: 100,
},
mockServerContext(),
),
).rejects.toThrow();
});
describe("observation time bounds", () => {
it.each([true, false])(
"should reject startTime filters with explicit type: %s",
async (includeType) => {
await expect(
handleListObservations(
{
...observationWindow,
fields: ["id"],
limit: 1,
filter: [
{
column: "startTime",
operator: ">=",
value: "2020-01-01T00:00:00Z",
...(includeType ? { type: "datetime" } : {}),
},
],
} as unknown as Parameters<typeof handleListObservations>[0],
mockServerContext(),
),
).rejects.toMatchObject({
issues: expect.arrayContaining([
expect.objectContaining({
path: ["filter", 0, "column"],
message: expect.stringContaining(
'Invalid observation filter column "startTime"',
),
}),
]),
});
},
);
it.each(["fromStartTime", "toStartTime"] as const)(
"should require %s for both tools even with compact fields",
async (missingBound) => {
const input = { ...observationWindow, [missingBound]: undefined };
await expect(
handleListObservations(
{ ...input, fields: ["id"], limit: 1 } as unknown as Parameters<
typeof handleListObservations
>[0],
mockServerContext(),
),
).rejects.toThrow(missingBound);
await expect(
handleGetObservation(
{
...input,
observationId: randomUUID(),
fields: ["id"],
} as unknown as Parameters<typeof handleGetObservation>[0],
mockServerContext(),
),
).rejects.toThrow(missingBound);
},
);
it.each([
["2026-01-01T00:00:00.000Z", "2026-01-01T00:00:00.000Z"],
["2026-01-02T00:00:00.000Z", "2026-01-01T00:00:00.000Z"],
["2026-01-01T00:00:00.000Z", "2026-04-01T00:00:00.001Z"],
])(
"should reject invalid periods %s to %s",
async (fromStartTime, toStartTime) => {
await expect(
handleListObservations(
{ fromStartTime, toStartTime, fields: ["id"], limit: 1 },
mockServerContext(),
),
).rejects.toThrow(/maximum range of 14 days/i);
await expect(
handleGetObservation(
{ fromStartTime, toStartTime, observationId: randomUUID() },
mockServerContext(),
),
).rejects.toThrow(/maximum range of 90 days/i);
},
);
it.each(["2026-01-15T00:00:00.001Z", "2026-01-15T23:59:59.000Z"])(
"should reject list periods exceeding 336 hours: %s",
async (toStartTime) => {
await expect(
handleListObservations(
{
fromStartTime: "2026-01-01T00:00:00.000Z",
toStartTime,
fields: ["id"],
limit: 1,
},
mockServerContext(),
),
).rejects.toThrow(/maximum range of 14 days \(336 elapsed hours\)/i);
},
);
it("should advertise required bounds for both tools", () => {
for (const tool of [getObservationTool, listObservationsTool]) {
expect(tool.inputSchema.required).toEqual(
expect.arrayContaining(["fromStartTime", "toStartTime"]),
);
}
});
});
maybeEventsTable("listObservations tool", () => {
it("should have readOnlyHint annotation", () => {
verifyToolAnnotations(listObservationsTool, { readOnlyHint: true });
});
it("should be available to in-app agent keys", async () => {
const context = mockServerContext({
inAppAgent: { permissions: "read" },
});
await expect(
toolRegistry.getEnabledTool(listObservationsTool.name, context),
).resolves.toBeDefined();
});
it("should expose object-shaped advanced filters in the tool schema", () => {
const properties = listObservationsTool.inputSchema.properties as Record<
string,
unknown
>;
expect(properties.isRootObservation).toMatchObject({ type: "boolean" });
const filterSchema = properties.filter as
| {
type?: string;
items?: {
type?: string;
required?: string[];
properties?: Record<
string,
{
const?: unknown;
enum?: unknown[];
type?: string;
}
>;
anyOf?: Array<{
type?: string;
properties?: Record<
string,
{
const?: unknown;
enum?: unknown[];
type?: string;
}
>;
required?: string[];
}>;
oneOf?: Array<{
type?: string;
properties?: Record<
string,
{
const?: unknown;
enum?: unknown[];
type?: string;
}
>;
required?: string[];
}>;
};
}
| undefined;
expect(filterSchema?.type).toBe("array");
expect(filterSchema?.items?.anyOf).toBeUndefined();
expect(filterSchema?.items?.oneOf).toBeUndefined();
expect(filterSchema?.items?.type).toBe("object");
expect(filterSchema?.items?.required).toEqual(
expect.arrayContaining(["column", "operator", "value"]),
);
expect(filterSchema?.items?.required).not.toContain("type");
expect(filterSchema?.items?.properties).toEqual(
expect.objectContaining({
column: expect.objectContaining({ type: "string" }),
operator: expect.objectContaining({ type: "string" }),
value: expect.any(Object),
type: expect.objectContaining({ type: "string" }),
key: expect.objectContaining({ type: "string" }),
}),
);
});
it("should filter by metadata advanced filters", async () => {
const { context, projectId } = await createMcpTestSetup();
const traceId = randomUUID();
const startTime = new Date();
const matchingObservation = createObservationEvent({
projectId,
traceId,
startTime,
name: `mcp-filter-metadata-match-${nanoid()}`,
metadata: { region: "us-east", tenant: "acme" },
});
const nonMatchingObservation = createObservationEvent({
projectId,
traceId,
startTime,
name: `mcp-filter-metadata-miss-${nanoid()}`,
metadata: { region: "eu-west", tenant: "acme" },
});
await createEventsCh([matchingObservation, nonMatchingObservation]);
const result = (await handleListObservations(
{
fromStartTime: new Date(startTime.getTime() - 1000).toISOString(),
toStartTime: new Date(startTime.getTime() + 1000).toISOString(),
filter: [
{
type: "stringOptions",
column: "id",
operator: "any of",
value: [matchingObservation.id, nonMatchingObservation.id],
},
{
type: "stringObject",
column: "metadata",
key: "region",
operator: "contains",
value: "us-",
},
],
fields: ["id", "name"],
limit: 100,
},
context,
)) as { data: Array<{ id: string; name: string; url: string }> };
expect(result.data).toEqual([
{
id: matchingObservation.id,
name: matchingObservation.name,
url: buildObservationUrl({
projectId,
traceId,
observationId: matchingObservation.id,
}),
},
]);
});
it("should list observations with compact default projection", async () => {
const { context, projectId } = await createMcpTestSetup();
const traceId = randomUUID();
const observation = createObservationEvent({
projectId,
traceId,
name: `mcp-list-default-${nanoid()}`,
input: "hidden input",
output: "hidden output",
metadata: { hidden: "metadata" },
});
await createEventsCh([observation]);
const result = (await handleListObservations(
{ ...observationWindow, traceId, limit: 100 },
context,
)) as {
data: Array<Record<string, unknown>>;
meta: Record<string, unknown>;
};
const createdObservation = result.data.find(
(item) => item.id === observation.id,
);
expect(createdObservation).toBeDefined();
expect(createdObservation).toMatchObject({
id: observation.id,
traceId,
name: observation.name,
type: "GENERATION",
level: "DEFAULT",
isRootObservation: true,
providedModelName: "gpt-4o-mini",
url: buildObservationUrl({
projectId,
traceId,
observationId: observation.id,
}),
});
expect(createdObservation?.input).toBeUndefined();
expect(createdObservation?.output).toBeUndefined();
expect(createdObservation?.metadata).toBeUndefined();
expect(result.meta).toBeDefined();
});
it("should list observations with an in-app agent context", async () => {
const { context, projectId } = await createMcpTestSetup();
const traceId = randomUUID();
const observation = createObservationEvent({
projectId,
traceId,
name: `mcp-list-in-app-agent-${nanoid()}`,
});
await createEventsCh([observation]);
const result = (await handleListObservations(
{ ...observationWindow, traceId, fields: ["id"], limit: 100 },
{ ...context, inAppAgent: { permissions: "read" } },
)) as { data: Array<{ id: string; url: string }> };
expect(result.data.map((item) => item.id)).toContain(observation.id);
expect(result.data.find((item) => item.id === observation.id)?.url).toBe(
buildObservationUrl({
projectId,
traceId,
observationId: observation.id,
}),
);
});
it("should project only requested fields", async () => {
const { context, projectId } = await createMcpTestSetup();
const traceId = randomUUID();
const observation = createObservationEvent({
projectId,
traceId,
name: `mcp-list-fields-${nanoid()}`,
});
await createEventsCh([observation]);
const result = (await handleListObservations(
{
...observationWindow,
traceId,
fields: ["id", "name", "type"],
limit: 100,
},
context,
)) as { data: Array<Record<string, unknown>> };
const createdObservation = result.data.find(
(item) => item.id === observation.id,
);
expect(createdObservation).toEqual({
id: observation.id,
name: observation.name,
type: "GENERATION",
url: buildObservationUrl({
projectId,
traceId,
observationId: observation.id,
}),
});
});
it("should include payload fields when requested with wildcard projection", async () => {
const { context, projectId } = await createMcpTestSetup();
const traceId = randomUUID();
const startTime = new Date();
const observation = createObservationEvent({
projectId,
traceId,
startTime,
name: `mcp-list-wildcard-${nanoid()}`,
input: "visible input",
output: "visible output",
metadata: { customer: "acme" },
});
await createEventsCh([observation]);
const result = (await handleListObservations(
{
traceId,
fields: ["*"],
fromStartTime: new Date(startTime.getTime() - 1000).toISOString(),
toStartTime: new Date(startTime.getTime() + 1000).toISOString(),
limit: 50,
},
context,
)) as { data: Array<Record<string, unknown>> };
const createdObservation = result.data.find(
(item) => item.id === observation.id,
);
expect(createdObservation?.input).toBe("visible input");
expect(createdObservation?.output).toBe("visible output");
expect(createdObservation?.metadata).toEqual({ customer: "acme" });
expect(createdObservation).toHaveProperty("userId");
});
it("should filter by advanced filters", async () => {
const { context, projectId } = await createMcpTestSetup();
const traceId = randomUUID();
const matchingUserId = `mcp-filter-user-${nanoid()}`;
await createEventsCh([
createObservationEvent({
projectId,
traceId,
userId: matchingUserId,
}),
createObservationEvent({
projectId,
traceId,
userId: `mcp-filter-user-miss-${nanoid()}`,
}),
]);
const result = (await handleListObservations(
{
traceId,
filter: [
{
type: "string",
column: "userId",
operator: "=",
value: matchingUserId,
},
],
fields: ["id", "userId"],
...observationWindow,
limit: 100,
},
context,
)) as { data: Array<{ id: string; userId: string | null; url: string }> };
expect(result.data).toHaveLength(1);
expect(result.data[0]).toEqual({
userId: matchingUserId,
id: expect.any(String),
url: buildObservationUrl({
projectId,
traceId,
observationId: result.data[0]!.id,
}),
});
});
it("should expose semantic roots across list and filter values", async () => {
const { context, projectId } = await createMcpTestSetup();
const traceId = randomUUID();
const parentlessRoot = createObservationEvent({ projectId, traceId });
const appRoot = createObservationEvent({
projectId,
traceId,
parentObservationId: randomUUID(),
isAppRoot: true,
});
const child = createObservationEvent({
projectId,
traceId,
parentObservationId: appRoot.id,
});
await createEventsCh([parentlessRoot, appRoot, child]);
const rootResult = (await handleListObservations(
{
traceId,
isRootObservation: true,
...observationWindow,
fields: ["id", "parentObservationId", "isRootObservation"],
limit: 100,
},
context,
)) as { data: Array<Record<string, unknown> & { id: string }> };
const getRootValues = async (isRootObservation?: boolean) =>
(await handleGetObservationFilterValues(
{
column: "isRootObservation",
isRootObservation,
limit: 100,
},
context,
)) as { values: Array<{ value: boolean; count?: number }> };
const [allRootValues, scopedRootValues] = await Promise.all([
getRootValues(),
getRootValues(true),
]);
expect(new Set(rootResult.data.map(({ id }) => id))).toEqual(
new Set([parentlessRoot.id, appRoot.id]),
);
expect(rootResult.data.find(({ id }) => id === appRoot.id)).toMatchObject(
{
parentObservationId: appRoot.parent_span_id,
isRootObservation: true,
},
);
expect(
allRootValues.values.map(({ value, count }) => [value, count]),
).toEqual([
[false, 1],
[true, 2],
]);
expect(
scopedRootValues.values.map(({ value, count }) => [value, count]),
).toEqual([[true, 2]]);
});
it("should match advanced input filters beyond the events_core truncation boundary", async () => {
const { context, projectId } = await createMcpTestSetup();
const traceId = randomUUID();
const startTime = new Date();
const needle = `needle-${nanoid()}`;
const matchingObservation = createObservationEvent({
projectId,
traceId,
startTime,
input: `${"x".repeat(250)} ${needle}`,
});
await createEventsCh([
matchingObservation,
createObservationEvent({
projectId,
traceId,
input: "short-input-without-match",
}),
]);
const result = (await handleListObservations(
{
traceId,
filter: [
{
type: "string",
column: "input",
operator: "matches",
value: needle,
},
],
fromStartTime: new Date(startTime.getTime() - 1000).toISOString(),
toStartTime: new Date(startTime.getTime() + 1000).toISOString(),
fields: ["id"],
limit: 100,
},
context,
)) as { data: Array<{ id: string; url: string }> };
expect(result.data).toEqual([
{
id: matchingObservation.id,
url: buildObservationUrl({
projectId,
traceId,
observationId: matchingObservation.id,
}),
},
]);
});
it("should require a time range for full io and metadata access", async () => {
const { context } = await createMcpTestSetup();
await expect(
handleListObservations(
{ fields: ["input"], limit: 100 } as unknown as Parameters<
typeof handleListObservations
>[0],
context,
),
).rejects.toMatchObject({
issues: expect.arrayContaining([
expect.objectContaining({ path: ["fromStartTime"] }),
expect.objectContaining({ path: ["toStartTime"] }),
]),
});
await expect(
handleListObservations(
{
fields: ["id"],
filter: [
{
type: "string",
column: "input",
operator: "matches",
value: "secret",
},
],
limit: 100,
} as unknown as Parameters<typeof handleListObservations>[0],
context,
),
).rejects.toMatchObject({
issues: expect.arrayContaining([
expect.objectContaining({ path: ["fromStartTime"] }),
expect.objectContaining({ path: ["toStartTime"] }),
]),
});
await expect(
handleListObservations(
{
fields: ["input"],
filter: [
{
type: "stringOptions",
column: "id",
operator: "any of",
value: [randomUUID()],
},
],
limit: 100,
} as unknown as Parameters<typeof handleListObservations>[0],
context,
),
).rejects.toMatchObject({
issues: expect.arrayContaining([
expect.objectContaining({ path: ["fromStartTime"] }),
expect.objectContaining({ path: ["toStartTime"] }),
]),
});
await expect(
handleListObservations(
{
traceId: randomUUID(),
fields: ["metadata"],
limit: 50,
} as unknown as Parameters<typeof handleListObservations>[0],
context,
),
).rejects.toMatchObject({
issues: expect.arrayContaining([
expect.objectContaining({ path: ["fromStartTime"] }),
expect.objectContaining({ path: ["toStartTime"] }),
]),
});
await expect(
handleListObservations(
{
fields: ["metadata"],
fromStartTime: "2026-01-01T00:00:00.000Z",
toStartTime: "2026-01-02T00:00:00.000Z",
limit: 100,
},
context,
),
).resolves.toMatchObject({ data: [] });
});
it("should limit date-scoped expensive observation access", async () => {
const { context } = await createMcpTestSetup();
await expect(
handleListObservations(
{
fields: ["input"],
fromStartTime: "2026-01-01T00:00:00.000Z",
toStartTime: "2026-01-16T00:00:00.000Z",
limit: 50,
},
context,
),
).rejects.toThrow(/maximum range of 14 days/i);
await expect(
handleListObservations(
{
fields: ["id"],
fromStartTime: "2026-01-01T00:00:00.000Z",
toStartTime: "2026-01-16T00:00:00.000Z",
filter: [
{
type: "string",
column: "input",
operator: "matches",
value: "needle",
},
],
limit: 50,
},
context,
),
).rejects.toThrow(/maximum range of 14 days/i);
await expect(
handleListObservations(
{
fields: ["input", "output"],
fromStartTime: "2026-01-01T00:00:00.000Z",
toStartTime: "2026-01-15T00:00:00.000Z",
limit: 51,
},
context,
),
).rejects.toThrow(/maximum limit of 50/i);
await expect(
handleListObservations(
{
fields: ["input"],
fromStartTime: "2026-01-01T00:00:00.000Z",
toStartTime: "2026-01-15T00:00:00.000Z",
limit: 50,
},
context,
),
).resolves.toMatchObject({ data: [] });
});
it.each([
{ traceId: randomUUID() },
{
filter: [
{
type: "string" as const,
column: "id",
operator: "=" as const,
value: randomUUID(),
},
],
},
])("should enforce the time range for scoped access: %j", async (scope) => {
const { context } = await createMcpTestSetup();
await expect(
handleListObservations(
{ ...scope, fields: ["input"], limit: 50 } as unknown as Parameters<
typeof handleListObservations
>[0],
context,
),
).rejects.toThrow(/fromStartTime.*toStartTime/is);
await expect(
handleListObservations(
{
...scope,
fields: ["metadata"],
fromStartTime: "2026-01-01T00:00:00.000Z",
toStartTime: "2026-01-16T00:00:00.000Z",
limit: 50,
},
context,
),
).rejects.toThrow(/maximum range of 14 days/i);
});
it("should allow an exact observation id filter with a time range", async () => {
const { context, projectId } = await createMcpTestSetup();
const startTime = new Date();
const observation = createObservationEvent({
projectId,
startTime,
input: "selective input",
});
await createEventsCh([observation]);
const result = (await handleListObservations(
{
fields: ["id", "input"],
fromStartTime: new Date(startTime.getTime() - 1000).toISOString(),
toStartTime: new Date(startTime.getTime() + 1000).toISOString(),
filter: [
{
type: "string",
column: "id",
operator: "=",
value: observation.id,
},
],
limit: 50,
},
context,
)) as { data: Array<{ id: string; input: string }> };
expect(result.data).toEqual([
{
id: observation.id,
input: "selective input",
url: buildObservationUrl({
projectId,
traceId: observation.trace_id,
observationId: observation.id,
}),
},
]);
});
it("should infer advanced filter type from the column", async () => {
const { context, projectId } = await createMcpTestSetup();
const traceId = randomUUID();
const matchingObservation = createObservationEvent({
projectId,
traceId,
name: `mcp-filter-infer-match-${nanoid()}`,
totalCost: 0.003,
});
await createEventsCh([
matchingObservation,
createObservationEvent({
projectId,
traceId,
name: `mcp-filter-infer-miss-${nanoid()}`,
totalCost: 0.001,
}),
]);
const result = (await handleListObservations(
{
traceId,
filter: [
{ column: "totalCost", operator: ">", value: 0.0029 } as any,
],
fields: ["id", "name"],
...observationWindow,
limit: 100,
},
context,
)) as { data: Array<{ id: string; name: string; url: string }> };
expect(result.data).toEqual([
{
id: matchingObservation.id,
name: matchingObservation.name,
url: buildObservationUrl({
projectId,
traceId,
observationId: matchingObservation.id,
}),
},
]);
});
it("should reject explicit advanced filters with a mismatched column type", async () => {
const { context } = await createMcpTestSetup();
await expect(
handleListObservations(
{
filter: [
{
type: "string",
column: "totalCost",
operator: "=",
value: "abc",
},
],
limit: 100,
...observationWindow,
},
context,
),
).rejects.toMatchObject({
issues: expect.arrayContaining([
expect.objectContaining({
path: ["filter", 0, "type"],
message: expect.stringMatching(/number/i),
}),
]),
});
});
it("should reject hidden/internal advanced filter columns", async () => {
const { context } = await createMcpTestSetup();
await expect(
handleListObservations(
{
filter: [
{
type: "arrayOptions",
column: "toolNames",
operator: "any of",
value: ["internal-tool"],
},
],
limit: 100,
...observationWindow,
},
context,
),
).rejects.toThrow(/getObservationFilterSchema/i);
await expect(
handleListObservations(
{
filter: [
{
type: "stringOptions",
column: "traceTags",
operator: "any of",
value: ["internal-name"],
},
],
limit: 100,
...observationWindow,
},
context,
),
).rejects.toThrow(/getObservationFilterSchema/i);
});
it("should allow the public tags advanced filter column", async () => {
const { context, projectId } = await createMcpTestSetup();
const traceId = randomUUID();
const matchingTag = `mcp-filter-tag-${nanoid()}`;
const matchingObservation = createObservationEvent({
projectId,
traceId,
name: `mcp-filter-tag-match-${nanoid()}`,
tags: [matchingTag],
});
await createEventsCh([
matchingObservation,
createObservationEvent({
projectId,
traceId,
name: `mcp-filter-tag-miss-${nanoid()}`,
tags: [`mcp-filter-tag-miss-${nanoid()}`],
}),
]);
const result = (await handleListObservations(
{
traceId,
filter: [
{
type: "arrayOptions",
column: "tags",
operator: "any of",
value: [matchingTag],
},
],
fields: ["id"],
...observationWindow,
limit: 100,
},
context,
)) as { data: Array<{ id: string; url: string }> };
expect(result.data).toEqual([
{
id: matchingObservation.id,
url: buildObservationUrl({
projectId,
traceId,
observationId: matchingObservation.id,
}),
},
]);
});
it("should return a cursor when more results are available", async () => {
const { context, projectId } = await createMcpTestSetup();
const traceId = randomUUID();
await createEventsCh([
createObservationEvent({
projectId,
traceId,
name: `mcp-cursor-1-${nanoid()}`,
startTime: new Date("2026-01-01T00:00:00.000Z"),
}),
createObservationEvent({
projectId,
traceId,
name: `mcp-cursor-2-${nanoid()}`,
startTime: new Date("2026-01-02T00:00:00.000Z"),
}),
]);
const result = (await handleListObservations(
{
traceId,
fields: ["id"],
limit: 1,
fromStartTime: "2026-01-01T00:00:00.000Z",
toStartTime: "2026-01-03T00:00:00.000Z",
},
context,
)) as { data: Array<{ id: string }>; meta: { cursor?: string } };
expect(result.data).toHaveLength(1);
expect(result.meta.cursor).toEqual(expect.any(String));
});
it("should reject invalid field projections", async () => {
const { context } = await createMcpTestSetup();
await expect(
handleListObservations(
{ ...observationWindow, fields: ["*", "id"], limit: 50 },
context,
),
).rejects.toThrow(/mixed wildcard projection/i);
await expect(
handleListObservations(
{ ...observationWindow, fields: ["unknownField"], limit: 50 },
context,
),
).rejects.toThrow(/getObservationFieldSchema/i);
});
it("should enforce max limit of 100", async () => {
const { context } = await createMcpTestSetup();
await expect(
handleListObservations({ ...observationWindow, limit: 101 }, context),
).rejects.toThrow(/<=100/i);
});
it("should use context.projectId for tenant isolation", async () => {
const { context: context1, projectId: projectId1 } =
await createMcpTestSetup();
const { context: context2 } = await createMcpTestSetup();
const traceId = randomUUID();
const observation = createObservationEvent({
projectId: projectId1,
traceId,
name: `mcp-list-isolation-${nanoid()}`,
});
await createEventsCh([observation]);
const result1 = (await handleListObservations(
{ ...observationWindow, traceId, fields: ["id"], limit: 100 },
context1,
)) as { data: Array<{ id: string }> };
const result2 = (await handleListObservations(
{ ...observationWindow, traceId, fields: ["id"], limit: 100 },
context2,
)) as { data: Array<{ id: string }> };
expect(result1.data.map((item) => item.id)).toContain(observation.id);
expect(result2.data).toEqual([]);
});
});
maybeEventsTable("queryMetrics tool", () => {
const metricsWindow = {
fromTimestamp: "2025-12-31T00:00:00.000Z",
toTimestamp: "2026-01-02T00:00:00.000Z",
};
const getMetricRows = (result: unknown) => {
expect(result).toMatchObject({ data: expect.any(Array) });
return Reflect.get(Object(result), "data");
};
it("should have readOnlyHint annotation", () => {
verifyToolAnnotations(queryMetricsTool, { readOnlyHint: true });
});
it("should expose object-shaped metrics query input in the tool schema", () => {
const properties = queryMetricsTool.inputSchema.properties as Record<
string,
unknown
>;
expect(properties.view).toBeDefined();
expect(properties.dimensions).toBeDefined();
expect(properties.metrics).toBeDefined();
expect(properties.filters).toBeDefined();
expect(properties.timeDimension).toBeDefined();
expect(properties.fromTimestamp).toBeDefined();
expect(properties.toTimestamp).toBeDefined();
expect(properties.orderBy).toBeDefined();
expect(properties.config).toBeDefined();
});
it("should query observations metrics for the current project", async () => {
const { context, projectId } = await createMcpTestSetup();
const traceId = randomUUID();
await createEventsCh(
Array.from({ length: 3 }, (_, index) =>
createObservationEvent({
projectId,
traceId,
name: `mcp-metrics-count-${index}-${nanoid()}`,
startTime: new Date(`2026-01-01T00:00:0${index}.000Z`),
}),
),
);
const rows = getMetricRows(
await handleQueryMetrics(
{
view: "observations",
metrics: [{ measure: "count", aggregation: "count" }],
filters: [
{
type: "string",
column: "traceId",
operator: "=",
value: traceId,
},
],
...metricsWindow,
} as unknown as Parameters<typeof handleQueryMetrics>[0],
context,
),
);
expect(rows).toHaveLength(1);
expect(Number(rows[0].count_count)).toBe(3);
});
it("should coerce an exact string tags filter", async () => {
const { context, projectId } = await createMcpTestSetup();
const traceId = randomUUID();
const matchingTag = `mcp-metrics-tag-${nanoid()}`;
await createEventsCh([
createObservationEvent({
projectId,
traceId,
tags: [matchingTag],
startTime: new Date("2026-01-01T00:00:00.000Z"),
}),
createObservationEvent({
projectId,
traceId,
tags: [`mcp-metrics-tag-miss-${nanoid()}`],
startTime: new Date("2026-01-01T00:01:00.000Z"),
}),
]);
const rows = getMetricRows(
await handleQueryMetrics(
{
view: "observations",
metrics: [{ measure: "count", aggregation: "count" }],
filters: [
{
type: "string",
column: "traceId",
operator: "=",
value: traceId,
},
{
type: "string",
column: "tags",
operator: "=",
value: matchingTag,
},
],
...metricsWindow,
} as unknown as Parameters<typeof handleQueryMetrics>[0],
context,
),
);
expect(rows).toHaveLength(1);
expect(Number(rows[0].count_count)).toBe(1);
});
it("should accept raw metric names in orderBy", async () => {
const { context, projectId } = await createMcpTestSetup();
const traceId = randomUUID();
const highCountName = `mcp-metrics-order-high-${nanoid()}`;
const lowCountName = `mcp-metrics-order-low-${nanoid()}`;
await createEventsCh([
...Array.from({ length: 3 }, (_, index) =>
createObservationEvent({
projectId,
traceId,
name: highCountName,
startTime: new Date(`2026-01-01T00:00:0${index}.000Z`),
}),
),
createObservationEvent({
projectId,
traceId,
name: lowCountName,
startTime: new Date("2026-01-01T00:01:00.000Z"),
}),
]);
const rows = getMetricRows(
await handleQueryMetrics(
{
view: "observations",
dimensions: [{ field: "name" }],
metrics: [{ measure: "count", aggregation: "count" }],
filters: [
{
type: "string",
column: "traceId",
operator: "=",
value: traceId,
},
],
orderBy: [{ field: "count", direction: "desc" }],
...metricsWindow,
} as unknown as Parameters<typeof handleQueryMetrics>[0],
context,
),
);
expect(rows).toHaveLength(2);
expect(rows[0].name).toBe(highCountName);
expect(Number(rows[0].count_count)).toBe(3);
expect(rows[1].name).toBe(lowCountName);
expect(Number(rows[1].count_count)).toBe(1);
});
it("should accept reversed metric aliases in orderBy", async () => {
const { context, projectId } = await createMcpTestSetup();
const traceId = randomUUID();
const highCostName = `mcp-metrics-cost-high-${nanoid()}`;
const lowCostName = `mcp-metrics-cost-low-${nanoid()}`;
await createEventsCh([
createObservationEvent({
projectId,
traceId,
name: highCostName,
totalCost: 0.02,
startTime: new Date("2026-01-01T00:00:00.000Z"),
}),
createObservationEvent({
projectId,
traceId,
name: lowCostName,
totalCost: 0.01,
startTime: new Date("2026-01-01T00:01:00.000Z"),
}),
]);
const rows = getMetricRows(
await handleQueryMetrics(
{
view: "observations",
dimensions: [{ field: "name" }],
metrics: [{ measure: "totalCost", aggregation: "sum" }],
filters: [
{
type: "string",
column: "traceId",
operator: "=",
value: traceId,
},
],
orderBy: [{ field: "totalCost_sum", direction: "desc" }],
...metricsWindow,
} as unknown as Parameters<typeof handleQueryMetrics>[0],
context,
),
);
expect(rows.map((row: { name: string }) => row.name)).toEqual([
highCostName,
lowCostName,
]);
});
it("should prefer dimension fields over matching raw metric names in orderBy", async () => {
const { context, projectId } = await createMcpTestSetup();
const traceId = randomUUID();
const scoreName = `mcp-metrics-score-value-order-${nanoid()}`;
const observationId = randomUUID();
await createEventsCh([
createEvent({
id: observationId,
span_id: observationId,
trace_id: traceId,
project_id: projectId,
name: `mcp-score-observation-${nanoid()}`,
start_time: Date.parse("2026-01-01T00:00:00.000Z") * 1000,
}),
]);
await createScoresCh([
...Array.from({ length: 3 }, () =>
createTraceScore({
id: randomUUID(),
project_id: projectId,
trace_id: traceId,
observation_id: observationId,
name: scoreName,
value: 1,
data_type: "NUMERIC",
timestamp: Date.parse("2026-01-01T00:00:00.000Z"),
}),
),
createTraceScore({
id: randomUUID(),
project_id: projectId,
trace_id: traceId,
observation_id: observationId,
name: scoreName,
value: 2,
data_type: "NUMERIC",
timestamp: Date.parse("2026-01-01T00:00:00.000Z"),
}),
]);
const rows = getMetricRows(
await handleQueryMetrics(
{
view: "scores-numeric",
dimensions: [{ field: "value" }],
metrics: [{ measure: "value", aggregation: "count" }],
filters: [
{
type: "string",
column: "name",
operator: "=",
value: scoreName,
},
],
orderBy: [{ field: "value", direction: "desc" }],
...metricsWindow,
} as unknown as Parameters<typeof handleQueryMetrics>[0],
context,
),
);
expect(rows).toHaveLength(2);
expect(Number(rows[0].value)).toBe(2);
expect(Number(rows[0].count_value)).toBe(1);
expect(Number(rows[1].value)).toBe(1);
expect(Number(rows[1].count_value)).toBe(3);
});
it("should reject internal sql field names in orderBy", async () => {
const { context } = await createMcpTestSetup();
await expect(
handleQueryMetrics(
{
view: "observations",
dimensions: [{ field: "name" }],
metrics: [{ measure: "count", aggregation: "count" }],
orderBy: [{ field: "observations.name", direction: "asc" }],
...metricsWindow,
} as unknown as Parameters<typeof handleQueryMetrics>[0],
context,
),
).rejects.toThrow(/Use returned metric aliases.*getMetricsSchema/i);
});
it("should return an invalid request for filters on pair-expanded dimensions", async () => {
const context = mockServerContext();
await expect(
handleQueryMetrics(
{
view: "observations",
metrics: [{ measure: "count", aggregation: "count" }],
filters: [
{
type: "string",
column: "usageType",
operator: "contains",
value: "cache",
},
],
...metricsWindow,
} as unknown as Parameters<typeof handleQueryMetrics>[0],
context,
),
).rejects.toMatchObject({
message: expect.stringContaining(
"Field 'usageType' cannot be used as a filter.",
),
});
});
it("should apply default row_limit of 100 when omitted", async () => {
const { context, projectId } = await createMcpTestSetup();
const traceId = randomUUID();
await createEventsCh(
Array.from({ length: 150 }, (_, index) =>
createObservationEvent({
projectId,
traceId,
name: `mcp-metrics-default-limit-${index}-${nanoid()}`,
startTime: new Date(Date.UTC(2026, 0, 1, 0, 0, 0) + index * 1000),
}),
),
);
const rows = getMetricRows(
await handleQueryMetrics(
{
view: "observations",
dimensions: [{ field: "name" }],
metrics: [{ measure: "count", aggregation: "count" }],
filters: [
{
type: "string",
column: "traceId",
operator: "=",
value: traceId,
},
],
...metricsWindow,
} as unknown as Parameters<typeof handleQueryMetrics>[0],
context,
),
);
expect(rows.length).toBeLessThanOrEqual(100);
});
it("should respect explicit row_limit", async () => {
const { context, projectId } = await createMcpTestSetup();
const traceId = randomUUID();
await createEventsCh(
Array.from({ length: 20 }, (_, index) =>
createObservationEvent({
projectId,
traceId,
name: `mcp-metrics-custom-limit-${index}-${nanoid()}`,
startTime: new Date(
`2026-01-01T00:00:${String(index).padStart(2, "0")}.000Z`,
),
}),
),
);
const rows = getMetricRows(
await handleQueryMetrics(
{
view: "observations",
dimensions: [{ field: "name" }],
metrics: [{ measure: "count", aggregation: "count" }],
filters: [
{
type: "string",
column: "traceId",
operator: "=",
value: traceId,
},
],
config: { row_limit: 5 },
...metricsWindow,
} as unknown as Parameters<typeof handleQueryMetrics>[0],
context,
),
);
expect(rows.length).toBeLessThanOrEqual(5);
});
it("should validate high-cardinality queries before applying default row_limit", async () => {
const { context } = await createMcpTestSetup();
await expect(
handleQueryMetrics(
{
view: "observations",
dimensions: [{ field: "traceId" }],
metrics: [{ measure: "count", aggregation: "count" }],
orderBy: [{ field: "count_count", direction: "desc" }],
...metricsWindow,
} as unknown as Parameters<typeof handleQueryMetrics>[0],
context,
),
).rejects.toThrow(
/require both 'config\.row_limit' and 'orderBy' with direction 'desc'/i,
);
});
});
maybeEventsTable("getMetricsSchema tool", () => {
const getMetricsSchemaViews = (result: unknown) => {
expect(result).toMatchObject({ views: expect.any(Object) });
return Reflect.get(Object(result), "views");
};
it("should have readOnlyHint annotation", () => {
verifyToolAnnotations(getMetricsSchemaTool, { readOnlyHint: true });
});
it("should return v2 metrics schema metadata", async () => {
const { context } = await createMcpTestSetup();
const result = await handleGetMetricsSchema({}, context);
const views = getMetricsSchemaViews(result);
expect(result).toMatchObject({
supportedViews: [
"observations",
"scores-numeric",
"scores-categorical",
"scores-boolean",
],
granularities: expect.arrayContaining(["day"]),
config: {
row_limit: { min: 1, max: 1000, default: 100 },
bins: { min: 1, max: 100 },
},
views: {
observations: {
filterableColumns: expect.arrayContaining([
{
column: "tags",
filterType: "arrayOptions",
operators: ["any of", "none of", "all of"],
},
{
column: "metadata",
filterType: "stringObject",
operators: [
"=",
"contains",
"does not contain",
"starts with",
"ends with",
"is set",
"is not set",
],
requiresKey: true,
},
]),
orderByFields: expect.arrayContaining([
"sum_totalCost",
"traceId",
"time_dimension",
]),
dimensions: {
traceId: {
highCardinality: true,
constraints: expect.any(String),
},
},
measures: {
count: {
validAggregations: expect.arrayContaining(["count"]),
},
},
},
"scores-boolean": {
dimensions: {
booleanValue: { type: "boolean" },
},
measures: {
value: {
validAggregations: expect.arrayContaining(["avg"]),
},
},
},
},
});
expect(views.observations.orderByFields).not.toContain(
"histogram_totalCost",
);
expect(views["scores-boolean"].dimensions.value).toBeUndefined();
expect(Reflect.get(Object(views), "traces")).toBeUndefined();
for (const viewName of Object.keys(views) as Array<
keyof (typeof viewDeclarations)["v2"]
>) {
const declaredDimensions = viewDeclarations.v2[viewName].dimensions;
const expectedFilterableDimensions = Object.entries(declaredDimensions)
.filter(
([, definition]) =>
definition.type !== undefined && !definition.pairExpand,
)
.map(([name]) => name);
const discoveredDimensions = views[viewName].filterableColumns
.map((column: { column: string }) => column.column)
.filter((column: string) => column in declaredDimensions);
expect(
discoveredDimensions.sort(),
`${viewName} MCP filter metadata must match filterable dimensions`,
).toEqual(expectedFilterableDimensions.sort());
}
});
it("should return one requested metrics view", async () => {
const { context } = await createMcpTestSetup();
const result = await handleGetMetricsSchema(
{ view: "scores-numeric" },
context,
);
const views = getMetricsSchemaViews(result);
expect(Object.keys(views)).toEqual(["scores-numeric"]);
});
});
maybeEventsTable("getObservation tool", () => {
it("should have readOnlyHint annotation", () => {
verifyToolAnnotations(getObservationTool, { readOnlyHint: true });
});
it("should be available to in-app agent keys", async () => {
const context = mockServerContext({
inAppAgent: { permissions: "read" },
});
await expect(
toolRegistry.getEnabledTool(getObservationTool.name, context),
).resolves.toBeDefined();
});
it("should fetch a single observation by id with compact default projection", async () => {
const { context, projectId } = await createMcpTestSetup();
const observation = createObservationEvent({
projectId,
name: `mcp-get-default-${nanoid()}`,
input: "hidden input",
});
await createEventsCh([observation]);
const result = (await handleGetObservation(
{
observationId: observation.id,
traceId: observation.trace_id,
fromStartTime: new Date(
Date.now() - 89 * 24 * 60 * 60 * 1000,
).toISOString(),
toStartTime: new Date(Date.now() + 1000).toISOString(),
},
context,
)) as Record<string, unknown>;
expect(result).toMatchObject({
id: observation.id,
traceId: observation.trace_id,
name: observation.name,
type: "GENERATION",
providedModelName: "gpt-4o-mini",
url: buildObservationUrl({
projectId,
traceId: observation.trace_id,
observationId: observation.id,
}),
});
expect(result.input).toBeUndefined();
expect(result.output).toBeUndefined();
expect(result.metadata).toBeUndefined();
});
it("should not fetch an observation outside the period or with a different traceId", async () => {
const { context, projectId } = await createMcpTestSetup();
const observation = createObservationEvent({ projectId });
await createEventsCh([observation]);
for (const scope of [
{ ...observationWindow, traceId: randomUUID() },
{
fromStartTime: "2026-01-01T00:00:00.000Z",
toStartTime: "2026-01-02T00:00:00.000Z",
},
]) {
await expect(
handleGetObservation(
{ observationId: observation.id, ...scope },
context,
),
).rejects.toThrow(/not found within the supplied period/i);
}
});
it("should enforce the plan data-access window for single observations", async () => {
const { context, projectId } = await createMcpTestSetup();
const now = Date.now();
const oldObservation = createObservationEvent({
projectId,
startTime: new Date(now - 40 * 24 * 60 * 60 * 1000),
});
const recentObservation = createObservationEvent({ projectId });
await createEventsCh([oldObservation, recentObservation]);
const period = {
fromStartTime: new Date(now - 60 * 24 * 60 * 60 * 1000).toISOString(),
toStartTime: new Date(now + 24 * 60 * 60 * 1000).toISOString(),
fields: ["id"],
};
const limitedContext = { ...context, plan: "cloud:hobby" as const };
await expect(
handleGetObservation(
{ ...period, observationId: oldObservation.id },
limitedContext,
),
).rejects.toThrow(/not found within the supplied period/i);
await expect(
handleGetObservation(
{ ...period, observationId: recentObservation.id },
limitedContext,
),
).resolves.toMatchObject({ id: recentObservation.id });
await expect(
handleGetObservation(
{ ...period, observationId: oldObservation.id },
{ ...context, plan: "oss" },
),
).resolves.toMatchObject({ id: oldObservation.id });
await expect(
handleGetObservation(
{
...period,
observationId: oldObservation.id,
toStartTime: new Date(now - 35 * 24 * 60 * 60 * 1000).toISOString(),
},
limitedContext,
),
).rejects.toThrow(/not found within the supplied period/i);
});
it("should fetch an observation with an in-app agent context", async () => {
const { context, projectId } = await createMcpTestSetup();
const observation = createObservationEvent({
projectId,
name: `mcp-get-in-app-agent-${nanoid()}`,
});
await createEventsCh([observation]);
const result = (await handleGetObservation(
{ ...observationWindow, observationId: observation.id, fields: ["id"] },
{ ...context, inAppAgent: { permissions: "read" } },
)) as Record<string, unknown>;
expect(result).toEqual({
id: observation.id,
url: buildObservationUrl({
projectId,
traceId: observation.trace_id,
observationId: observation.id,
}),
});
});
it("should return requested fields for a single observation", async () => {
const { context, projectId } = await createMcpTestSetup();
const observation = createObservationEvent({
projectId,
name: `mcp-get-fields-${nanoid()}`,
metadata: { customer: "acme" },
});
await createEventsCh([observation]);
const result = (await handleGetObservation(
{
...observationWindow,
observationId: observation.id,
fields: ["id", "metadata"],
},
context,
)) as Record<string, unknown>;
expect(result).toEqual({
id: observation.id,
metadata: { customer: "acme" },
url: buildObservationUrl({
projectId,
traceId: observation.trace_id,
observationId: observation.id,
}),
});
});
it("should reject missing observations", async () => {
const { context } = await createMcpTestSetup();
const observationId = randomUUID();
await expect(
handleGetObservation({ ...observationWindow, observationId }, context),
).rejects.toThrow(new RegExp(`Observation ${observationId} not found`));
});
it("should not fetch cross-project observations", async () => {
const { projectId: projectId1 } = await createMcpTestSetup();
const { context: context2 } = await createMcpTestSetup();
const observation = createObservationEvent({
projectId: projectId1,
name: `mcp-get-isolation-${nanoid()}`,
});
await createEventsCh([observation]);
await expect(
handleGetObservation(
{ ...observationWindow, observationId: observation.id },
context2,
),
).rejects.toThrow(/not found/i);
});
});
maybeEventsTable("getObservationFilterValues tool", () => {
it("should have readOnlyHint annotation", () => {
verifyToolAnnotations(getObservationFilterValuesTool, {
readOnlyHint: true,
});
});
it("should expose semantic-root inputs", () => {
const properties = getObservationFilterValuesTool.inputSchema
.properties as Record<string, unknown>;
expect(properties).toMatchObject({
column: { enum: expect.arrayContaining(["isRootObservation"]) },
isRootObservation: { type: "boolean" },
});
});
it("should be available to in-app agent keys", async () => {
const context = mockServerContext({
inAppAgent: { permissions: "read" },
});
await expect(
toolRegistry.getEnabledTool(
getObservationFilterValuesTool.name,
context,
),
).resolves.toBeDefined();
});
it("should return values for a dynamic filter column", async () => {
const { context, projectId } = await createMcpTestSetup();
const uniqueModel = `mcp-model-${nanoid()}`;
await createEventsCh([
createObservationEvent({
projectId,
name: `mcp-filter-values-${nanoid()}`,
providedModelName: uniqueModel,
}),
createObservationEvent({
projectId,
name: `mcp-filter-values-${nanoid()}`,
providedModelName: uniqueModel,
}),
]);
const result = (await handleGetObservationFilterValues(
{ column: "providedModelName", limit: 100 },
context,
)) as {
column: string;
values: Array<{ value: string; count?: number }>;
meta: Record<string, unknown>;
};
expect(result.column).toBe("providedModelName");
expect(result.values).toEqual(
expect.arrayContaining([
expect.objectContaining({ value: uniqueModel, count: 2 }),
]),
);
expect(result.meta).toBeDefined();
});
it("should return string examples and numeric ranges for filter columns", async () => {
const { context, projectId } = await createMcpTestSetup();
const uniqueSessionId = `mcp-session-${nanoid()}`;
const lowTotalCost = 0.1;
const highTotalCost = 0.3;
await createEventsCh([
createObservationEvent({
projectId,
name: `mcp-filter-examples-${nanoid()}`,
sessionId: uniqueSessionId,
totalCost: lowTotalCost,
}),
createObservationEvent({
projectId,
name: `mcp-filter-examples-${nanoid()}`,
sessionId: uniqueSessionId,
totalCost: highTotalCost,
}),
]);
const sessionResult = await handleGetObservationFilterValues(
{ column: "sessionId", limit: 100 },
context,
);
expect(sessionResult).toEqual(
expect.objectContaining({
type: "VALUES",
column: "sessionId",
values: expect.arrayContaining([
expect.objectContaining({ value: uniqueSessionId, count: 2 }),
]),
}),
);
const costResult = await handleGetObservationFilterValues(
{ column: "totalCost", limit: 100 },
context,
);
expect(costResult).toEqual(
expect.objectContaining({
type: "RANGE",
column: "totalCost",
range: expect.objectContaining({
count: 2,
min: expect.closeTo(lowTotalCost),
max: expect.closeTo(highTotalCost),
avg: expect.closeTo(0.2),
}),
}),
);
});
it("should map public tags column to traceTags option source", async () => {
const { context, projectId } = await createMcpTestSetup();
const uniqueTag = `mcp-tag-${nanoid()}`;
await createEventsCh([
createObservationEvent({
projectId,
name: `mcp-filter-tags-${nanoid()}`,
tags: [uniqueTag],
}),
]);
const result = (await handleGetObservationFilterValues(
{ column: "tags", limit: 100 },
context,
)) as { column: string; values: Array<{ value: string }> };
expect(result.column).toBe("tags");
expect(result.values).toEqual(
expect.arrayContaining([expect.objectContaining({ value: uniqueTag })]),
);
});
it("should return boolean values for hasParentObservation with counts", async () => {
const { context, projectId } = await createMcpTestSetup();
await createEventsCh([
createObservationEvent({
projectId,
name: `mcp-filter-has-parent-${nanoid()}`,
parentObservationId: randomUUID(),
}),
createObservationEvent({
projectId,
name: `mcp-filter-root-${nanoid()}`,
parentObservationId: null,
}),
]);
const result = (await handleGetObservationFilterValues(
{ column: "hasParentObservation", limit: 100 },
context,
)) as {
column: string;
values: Array<{ value: boolean; count?: number }>;
};
expect(result.column).toBe("hasParentObservation");
expect(result.values).toEqual([
expect.objectContaining({ value: false, count: 1 }),
expect.objectContaining({ value: true, count: 1 }),
]);
});
it("should preserve backend order and paginate filter values with an opaque cursor", async () => {
const { context, projectId } = await createMcpTestSetup();
const topName = `mcp-filter-page-top-${nanoid()}`;
const nextName = `mcp-filter-page-next-${nanoid()}`;
const lastName = `mcp-filter-page-last-${nanoid()}`;
await createEventsCh([
createObservationEvent({ projectId, name: topName }),
createObservationEvent({ projectId, name: topName }),
createObservationEvent({ projectId, name: topName }),
createObservationEvent({ projectId, name: nextName }),
createObservationEvent({ projectId, name: nextName }),
createObservationEvent({ projectId, name: lastName }),
]);
const firstPage = (await handleGetObservationFilterValues(
{ column: "name", limit: 2 },
context,
)) as {
values: Array<{ value: string; count?: number }>;
meta: { cursor?: string };
};
expect(firstPage.values).toEqual([
{ value: topName, count: 3 },
{ value: nextName, count: 2 },
]);
expect(firstPage.meta.cursor).toEqual(expect.any(String));
const secondPage = (await handleGetObservationFilterValues(
{ column: "name", limit: 2, cursor: firstPage.meta.cursor },
context,
)) as {
values: Array<{ value: string; count?: number }>;
};
expect(secondPage.values).toEqual([{ value: lastName, count: 1 }]);
});
it("should reject invalid cursors and unavailable columns", async () => {
const { context } = await createMcpTestSetup();
await expect(
handleGetObservationFilterValues(
{ column: "name", limit: 100, cursor: "not-base64-json" },
context,
),
).rejects.toThrow(/invalid cursor format/i);
await expect(
handleGetObservationFilterValues(
{ column: "input", limit: 100 } as any,
context,
),
).rejects.toMatchObject({
issues: expect.arrayContaining([
expect.objectContaining({ path: ["column"], code: "invalid_value" }),
]),
});
});
});
describe("listScores tool", () => {
it("should have readOnlyHint annotation", () => {
verifyToolAnnotations(listScoresTool, { readOnlyHint: true });
});
it("should return v3-shaped scores with polymorphic value, subject, and url", async () => {
const { context, projectId } = await createMcpTestSetup();
const numericScore = createTraceScore({
project_id: projectId,
id: randomUUID(),
name: `mcp-score-${nanoid()}`,
data_type: "NUMERIC",
value: 0.9,
});
const booleanScore = createTraceScore({
project_id: projectId,
id: randomUUID(),
name: `mcp-score-${nanoid()}`,
data_type: "BOOLEAN",
value: 1,
string_value: "True",
});
const sessionScore = createSessionScore({
project_id: projectId,
id: randomUUID(),
name: `mcp-score-${nanoid()}`,
data_type: "CATEGORICAL",
value: 0,
string_value: "good",
});
await createScoresCh([numericScore, booleanScore, sessionScore]);
const result = (await handleListScores(
{
limit: 10,
scoreIds: [numericScore.id, booleanScore.id, sessionScore.id],
dataType: ["NUMERIC"],
},
context,
)) as any;
expect(Object.keys(result).sort()).toEqual(["data", "meta"]);
expect(result.meta).toEqual({ limit: 10 });
expect(result.data).toEqual([
expect.objectContaining({
id: numericScore.id,
dataType: "NUMERIC",
value: 0.9,
comment: numericScore.comment,
subject: { kind: "trace", id: numericScore.trace_id },
url: buildTraceUrl({
projectId,
traceId: numericScore.trace_id!,
}),
}),
]);
const booleanResult = (await handleListScores(
{ limit: 10, scoreIds: [booleanScore.id] },
context,
)) as any;
expect(booleanResult.data).toEqual([
expect.objectContaining({
id: booleanScore.id,
dataType: "BOOLEAN",
value: true,
}),
]);
const sessionResult = (await handleListScores(
{ limit: 10, scoreIds: [sessionScore.id] },
context,
)) as any;
expect(sessionResult.data).toEqual([
expect.objectContaining({
id: sessionScore.id,
dataType: "CATEGORICAL",
value: "good",
subject: { kind: "session", id: sessionScore.session_id },
url: buildSessionUrl({
projectId,
sessionId: sessionScore.session_id!,
}),
}),
]);
});
it("should link experiment-subject scores to the experiment results page", async () => {
const { context, projectId } = await createMcpTestSetup();
const runScore = createDatasetRunScore({
project_id: projectId,
id: randomUUID(),
name: `mcp-run-score-${nanoid()}`,
data_type: "NUMERIC",
value: 0.7,
});
await createScoresCh([runScore]);
const result = (await handleListScores(
{ limit: 10, scoreIds: [runScore.id] },
context,
)) as any;
expect(result.data).toEqual([
expect.objectContaining({
id: runScore.id,
subject: { kind: "experiment", id: runScore.dataset_run_id },
url: buildExperimentUrl({
projectId,
experimentId: runScore.dataset_run_id!,
}),
}),
]);
});
it("should paginate with an opaque cursor", async () => {
const { context, projectId } = await createMcpTestSetup();
const name = `mcp-cursor-score-${nanoid()}`;
const now = Date.now();
const scores = [0, 1, 2].map((i) =>
createTraceScore({
project_id: projectId,
id: randomUUID(),
name,
data_type: "NUMERIC",
value: i,
timestamp: now - i * 60_000,
}),
);
await createScoresCh(scores);
const firstPage = (await handleListScores(
{ limit: 2, name: [name] },
context,
)) as any;
expect(firstPage.data).toHaveLength(2);
expect(firstPage.meta.limit).toBe(2);
expect(firstPage.meta.cursor).toEqual(expect.any(String));
const secondPage = (await handleListScores(
{ limit: 2, name: [name], cursor: firstPage.meta.cursor },
context,
)) as any;
expect(secondPage.data).toHaveLength(1);
expect(secondPage.meta.cursor).toBeUndefined();
const seenIds = [...firstPage.data, ...secondPage.data].map(
(score: { id: string }) => score.id,
);
expect(seenIds.sort()).toEqual(scores.map((s) => s.id).sort());
await expect(
handleListScores(
{ limit: 2, name: [name], cursor: "not-base64-json" },
context,
),
).rejects.toThrow(/invalid cursor format/i);
// Decodable JSON with a mismatched cursor schema (e.g. future version)
// must surface the same error, not a raw schema failure.
const wrongVersionCursor = Buffer.from(
JSON.stringify({ v: 2, lastId: "x" }),
).toString("base64url");
await expect(
handleListScores(
{ limit: 2, name: [name], cursor: wrongVersionCursor },
context,
),
).rejects.toThrow(/invalid cursor format/i);
});
it("should enforce v3 cross-field validation and reject dropped v2 params", async () => {
const { context } = await createMcpTestSetup();
await expect(
handleListScores({ limit: 10, value: ["0.5"] }, context),
).rejects.toThrow(/value filter requires a single dataType/i);
// Number("") === 0: an empty placeholder must not become a value=0 filter.
await expect(
handleListScores(
{ limit: 10, value: [""], dataType: ["NUMERIC"] },
context,
),
).rejects.toThrow(/value filter entries must be non-empty/i);
await expect(
handleListScores(
{ limit: 10, value: [" "], dataType: ["NUMERIC"] },
context,
),
).rejects.toThrow(/value filter entries must be non-empty/i);
await expect(
handleListScores({ limit: 10, valueMin: 0.5 }, context),
).rejects.toThrow(/valueMin and valueMax require dataType=NUMERIC/i);
await expect(
handleListScores({ limit: 10, observationId: [randomUUID()] }, context),
).rejects.toThrow(/observationId filter requires traceId/i);
await expect(
handleListScores(
{ limit: 10, traceId: [randomUUID()], sessionId: [randomUUID()] },
context,
),
).rejects.toThrow(/At most one of traceId, sessionId, experimentId/i);
// v2-only params (userId, traceTags, page, fields, filter) are rejected
// by the strict schema instead of being silently ignored.
await expect(
handleListScores({ limit: 10, userId: "user-1" } as any, context),
).rejects.toMatchObject({
issues: expect.arrayContaining([
expect.objectContaining({
code: "unrecognized_keys",
keys: ["userId"],
}),
]),
});
await expect(
handleListScores({ limit: 10, traceTags: ["tag"] } as any, context),
).rejects.toMatchObject({
issues: expect.arrayContaining([
expect.objectContaining({
code: "unrecognized_keys",
keys: ["traceTags"],
}),
]),
});
await expect(
handleListScores({ limit: 10, page: 1 } as any, context),
).rejects.toMatchObject({
issues: expect.arrayContaining([
expect.objectContaining({
code: "unrecognized_keys",
keys: ["page"],
}),
]),
});
});
});
describe("getScore tool", () => {
it("should have readOnlyHint annotation", () => {
verifyToolAnnotations(getScoreTool, { readOnlyHint: true });
});
it("should fetch a score by id", async () => {
const { context, projectId } = await createMcpTestSetup();
const score = createTraceScore({
project_id: projectId,
id: randomUUID(),
name: `mcp-get-score-${nanoid()}`,
data_type: "NUMERIC",
value: 0.8,
});
await createScoresCh([score]);
const result = await handleGetScore({ scoreId: score.id }, context);
expect(result).toMatchObject({
id: score.id,
name: score.name,
dataType: "NUMERIC",
value: 0.8,
comment: score.comment,
subject: { kind: "trace", id: score.trace_id },
url: buildTraceUrl({ projectId, traceId: score.trace_id! }),
});
});
it("should reject missing and cross-project scores", async () => {
const { projectId } = await createMcpTestSetup();
const { context: otherContext } = await createMcpTestSetup();
const score = createTraceScore({
project_id: projectId,
id: randomUUID(),
});
await createScoresCh([score]);
await expect(
handleGetScore({ scoreId: randomUUID() }, otherContext),
).rejects.toThrow(/Score not found/i);
await expect(
handleGetScore({ scoreId: score.id }, otherContext),
).rejects.toThrow(/Score not found/i);
});
});
describe("createScore tool", () => {
it("should have destructiveHint annotation", () => {
verifyToolAnnotations(createScoreTool, { destructiveHint: true });
});
it("should create a score using v1 route semantics", async () => {
const { context, projectId, apiKeyId } = await createMcpTestSetup();
const scoreId = randomUUID();
const traceId = randomUUID();
const result = await handleCreateScore(
{
id: scoreId,
traceId,
name: `mcp-create-score-${nanoid(8)}`,
value: 1,
dataType: "NUMERIC",
environment: "default",
source: "API",
},
context,
);
expect(result).toEqual({
id: scoreId,
url: buildTraceUrl({ projectId, traceId }),
});
await expect(
verifyAuditLog({
projectId,
apiKeyId,
resourceType: "score",
resourceId: scoreId,
action: "create",
}),
).resolves.toMatchObject({
resourceType: "score",
resourceId: scoreId,
action: "create",
});
});
});
describe("listScoreConfigs tool", () => {
it("should have readOnlyHint annotation", () => {
verifyToolAnnotations(listScoreConfigsTool, { readOnlyHint: true });
});
it("should return paginated score configs for the current project", async () => {
const { context, projectId } = await createMcpTestSetup();
const { context: otherContext, projectId: otherProjectId } =
await createMcpTestSetup();
const name = `mcp-config-${nanoid()}`;
await prisma.scoreConfig.createMany({
data: [
{
id: randomUUID(),
projectId,
name,
dataType: ScoreConfigDataType.NUMERIC,
minValue: 0,
maxValue: 1,
},
{
id: randomUUID(),
projectId: otherProjectId,
name,
dataType: ScoreConfigDataType.NUMERIC,
minValue: 0,
maxValue: 1,
},
],
});
const result = (await handleListScoreConfigs(
{ limit: 10, page: 1 },
context,
)) as any;
const otherResult = (await handleListScoreConfigs(
{ limit: 10, page: 1 },
otherContext,
)) as any;
expect(Object.keys(result).sort()).toEqual(["data", "meta"]);
expect(result.data).toEqual(
expect.arrayContaining([expect.objectContaining({ projectId, name })]),
);
expect(result.data).not.toEqual(
expect.arrayContaining([
expect.objectContaining({ projectId: otherProjectId, name }),
]),
);
expect(otherResult.data).toEqual(
expect.arrayContaining([
expect.objectContaining({ projectId: otherProjectId, name }),
]),
);
});
it("should paginate score configs deterministically when createdAt timestamps tie", async () => {
const { context, projectId } = await createMcpTestSetup();
const sharedCreatedAt = new Date("2100-05-12T00:00:00.000Z");
const configIDs = [randomUUID(), randomUUID(), randomUUID()];
await prisma.scoreConfig.createMany({
data: configIDs.map((id, index) => ({
id,
projectId,
name: `mcp-tie-score-config-${index}-${id.slice(0, 8)}`,
dataType: ScoreConfigDataType.NUMERIC,
minValue: index,
maxValue: index + 1,
createdAt: sharedCreatedAt,
updatedAt: sharedCreatedAt,
})),
});
const firstPage = (await handleListScoreConfigs(
{ limit: 2, page: 1 },
context,
)) as any;
const secondPage = (await handleListScoreConfigs(
{ limit: 2, page: 2 },
context,
)) as any;
const tiedIDs = [...firstPage.data, ...secondPage.data]
.filter((config) => config.id && configIDs.includes(config.id))
.map((config) => config.id);
expect(tiedIDs).toEqual(configIDs.slice().sort());
expect(new Set(tiedIDs).size).toBe(configIDs.length);
});
});
describe("getScoreConfig tool", () => {
it("should have readOnlyHint annotation", () => {
verifyToolAnnotations(getScoreConfigTool, { readOnlyHint: true });
});
it("should fetch a score config by id", async () => {
const { context, projectId } = await createMcpTestSetup();
const config = await prisma.scoreConfig.create({
data: {
id: randomUUID(),
projectId,
name: `mcp-get-${nanoid(8)}`,
dataType: ScoreConfigDataType.BOOLEAN,
categories: [
{ label: "True", value: 1 },
{ label: "False", value: 0 },
],
},
});
const result = await handleGetScoreConfig(
{ configId: config.id },
context,
);
expect(result).toMatchObject({
id: config.id,
name: config.name,
dataType: ScoreConfigDataType.BOOLEAN,
});
});
it("should reject missing and cross-project score configs", async () => {
const { projectId } = await createMcpTestSetup();
const { context: otherContext } = await createMcpTestSetup();
const config = await prisma.scoreConfig.create({
data: {
id: randomUUID(),
projectId,
name: `mcp-cross-${nanoid(8)}`,
dataType: ScoreConfigDataType.TEXT,
},
});
await expect(
handleGetScoreConfig({ configId: randomUUID() }, otherContext),
).rejects.toThrow(/Score config not found/i);
await expect(
handleGetScoreConfig({ configId: config.id }, otherContext),
).rejects.toThrow(/Score config not found/i);
});
});
describe("createScoreConfig tool", () => {
it("should have destructiveHint annotation", () => {
verifyToolAnnotations(createScoreConfigTool, { destructiveHint: true });
});
it("should describe allowed score config name characters in the input schema", () => {
expect(
containsStringValue(
createScoreConfigTool.inputSchema,
"Allowed characters: letters, numbers, spaces, underscores, periods, parentheses, and hyphens.",
),
).toBe(true);
});
it.each([
[
"numeric",
{
name: `mcp-num-${nanoid(8)}`,
dataType: "NUMERIC" as const,
numericMinValue: 0,
numericMaxValue: 1,
},
],
[
"categorical",
{
name: `mcp-cat-${nanoid(8)}`,
dataType: "CATEGORICAL" as const,
categoricalCategories: [
{ label: "High", value: 1 },
{ label: "Low", value: 0 },
],
},
],
[
"text",
{
name: `mcp-text-${nanoid(8)}`,
dataType: "TEXT" as const,
},
],
])("should create %s score configs", async (_type, input) => {
const { context, projectId, apiKeyId } = await createMcpTestSetup();
const result = (await handleCreateScoreConfig(
input as unknown as Parameters<typeof handleCreateScoreConfig>[0],
context,
)) as any;
expect(result).toMatchObject({
projectId,
name: input.name,
dataType: input.dataType,
});
await expect(
verifyAuditLog({
projectId,
resourceType: "scoreConfig",
resourceId: result.id,
action: "create",
apiKeyId,
}),
).resolves.toMatchObject({ action: "create" });
});
it("should create boolean configs with inferred categories", async () => {
const { context } = await createMcpTestSetup();
const result = await handleCreateScoreConfig(
{
name: `mcp-bool-${nanoid(8)}`,
dataType: "BOOLEAN",
} as unknown as Parameters<typeof handleCreateScoreConfig>[0],
context,
);
expect(result).toMatchObject({
dataType: ScoreConfigDataType.BOOLEAN,
categories: [
{ label: "True", value: 1 },
{ label: "False", value: 0 },
],
});
});
it("should reject invalid categories", async () => {
const { context } = await createMcpTestSetup();
await expect(
handleCreateScoreConfig(
{
name: `mcp-invalid-${nanoid(8)}`,
dataType: "CATEGORICAL",
categoricalCategories: [
{ label: "Duplicate", value: 1 },
{ label: "Duplicate", value: 2 },
],
} as unknown as Parameters<typeof handleCreateScoreConfig>[0],
context,
),
).rejects.toThrow(/Category labels must be unique/i);
});
it("should validate normalized numeric range fields even when another field is invalid", async () => {
const { context } = await createMcpTestSetup();
await expect(
handleCreateScoreConfig(
{
name: "invalid/name",
dataType: "NUMERIC",
numericMinValue: "not-a-number",
} as any,
context,
),
).rejects.toThrow(/minValue/i);
});
it("should reject invalid name characters at runtime despite the relaxed advertised schema", async () => {
const { context } = await createMcpTestSetup();
await expect(
handleCreateScoreConfig(
{
name: "css-injection}*{background:red}/*",
dataType: "TEXT",
} as unknown as Parameters<typeof handleCreateScoreConfig>[0],
context,
),
).rejects.toThrow(/invalid characters/i);
});
});
describe("updateScoreConfig tool", () => {
it("should have destructiveHint annotation", () => {
verifyToolAnnotations(updateScoreConfigTool, { destructiveHint: true });
});
it("should describe allowed score config name characters in the input schema", () => {
expect(
containsStringValue(
updateScoreConfigTool.inputSchema,
"Allowed characters: letters, numbers, spaces, underscores, periods, parentheses, and hyphens.",
),
).toBe(true);
});
it("should not expose archive state in the input schema", () => {
expect(
containsPropertyName(updateScoreConfigTool.inputSchema, "isArchived"),
).toBe(false);
});
it("should update allowed fields and write an audit log", async () => {
const { context, projectId, apiKeyId } = await createMcpTestSetup();
const config = await prisma.scoreConfig.create({
data: {
id: randomUUID(),
projectId,
name: `mcp-update-${nanoid(8)}`,
dataType: ScoreConfigDataType.NUMERIC,
minValue: 0,
maxValue: 1,
},
});
const result = await handleUpdateScoreConfig(
{
configId: config.id,
name: "mcp-updated",
description: "Updated through MCP",
numericMinValue: -1,
} as unknown as Parameters<typeof handleUpdateScoreConfig>[0],
context,
);
expect(result).toMatchObject({
id: config.id,
name: "mcp-updated",
description: "Updated through MCP",
minValue: -1,
});
await expect(
verifyAuditLog({
projectId,
resourceType: "scoreConfig",
resourceId: config.id,
action: "update",
apiKeyId,
}),
).resolves.toMatchObject({ action: "update" });
});
it("should reject empty update bodies", async () => {
const { context } = await createMcpTestSetup();
await expect(
handleUpdateScoreConfig({ configId: randomUUID() }, context),
).rejects.toThrow(/Request body cannot be empty/i);
});
it("should validate normalized update fields even when another field is invalid", async () => {
const { context } = await createMcpTestSetup();
await expect(
handleUpdateScoreConfig(
{
configId: randomUUID(),
name: "invalid/name",
numericMinValue: "not-a-number",
} as any,
context,
),
).rejects.toThrow(/minValue/i);
});
it("should reject invalid name characters at runtime despite the relaxed advertised schema", async () => {
const { context } = await createMcpTestSetup();
await expect(
handleUpdateScoreConfig(
{
configId: randomUUID(),
name: "css-injection}*{background:red}/*",
},
context,
),
).rejects.toThrow(/invalid characters/i);
});
it("should not archive score configs through update", async () => {
const { context, projectId } = await createMcpTestSetup();
const config = await prisma.scoreConfig.create({
data: {
id: randomUUID(),
projectId,
name: `mcp-update-no-archive-${nanoid(8)}`,
dataType: ScoreConfigDataType.NUMERIC,
minValue: 0,
maxValue: 1,
},
});
const result = await handleUpdateScoreConfig(
{
configId: config.id,
name: "mcp-still-active",
isArchived: true,
} as unknown as Parameters<typeof handleUpdateScoreConfig>[0],
context,
);
expect(result).toMatchObject({
id: config.id,
name: "mcp-still-active",
isArchived: false,
});
});
});
describe("deleteScoreConfig tool", () => {
it("should have destructiveHint annotation", () => {
verifyToolAnnotations(deleteScoreConfigTool, { destructiveHint: true });
});
it("should archive the score config", async () => {
const { context, projectId, apiKeyId } = await createMcpTestSetup();
const config = await prisma.scoreConfig.create({
data: {
id: randomUUID(),
projectId,
name: `mcp-delete-config-${nanoid(8)}`,
dataType: ScoreConfigDataType.NUMERIC,
minValue: 0,
maxValue: 1,
},
});
const result = await handleDeleteScoreConfig(
{ configId: config.id },
context,
);
expect(result).toMatchObject({
id: config.id,
isArchived: true,
});
await expect(
verifyAuditLog({
projectId,
resourceType: "scoreConfig",
resourceId: config.id,
action: "update",
apiKeyId,
}),
).resolves.toMatchObject({ action: "update" });
});
});
describe("getPrompt tool", () => {
it("should have readOnlyHint annotation", () => {
verifyToolAnnotations(getPromptTool, { readOnlyHint: true });
});
it("should be available to in-app agent keys", async () => {
const context = mockServerContext({
inAppAgent: { permissions: "read" },
});
await expect(
toolRegistry.getEnabledTool(getPromptTool.name, context),
).resolves.toBeDefined();
});
it("should fetch prompt by name only (defaults to latest label)", async () => {
const { context, projectId } = await createMcpTestSetup();
const promptName = `test-prompt-${nanoid()}`;
await createPromptInDb({
name: promptName,
prompt: "Production prompt",
projectId,
labels: ["production"],
version: 1,
});
await createPromptInDb({
name: promptName,
prompt: "Latest prompt",
projectId,
labels: ["latest"],
version: 2,
});
const result = (await handleGetPrompt({ name: promptName }, context)) as {
name: string;
version: number;
prompt: string;
labels: string[];
url: string;
};
expect(result.name).toBe(promptName);
expect(result.version).toBe(2);
expect(result.prompt).toBe("Latest prompt");
expect(result.labels).toContain("latest");
expect(result.url).toBe(
buildPromptUrl({ projectId, name: promptName, version: 2 }),
);
});
it("should fetch production prompt when production label is explicit", async () => {
const { context, projectId } = await createMcpTestSetup();
const promptName = `test-prompt-${nanoid()}`;
await createPromptInDb({
name: promptName,
prompt: "Production prompt",
projectId,
labels: ["production"],
version: 1,
});
await createPromptInDb({
name: promptName,
prompt: "Latest prompt",
projectId,
labels: ["latest"],
version: 2,
});
const result = (await handleGetPrompt(
{ name: promptName, label: "production" },
context,
)) as {
version: number;
prompt: string;
labels: string[];
};
expect(result.version).toBe(1);
expect(result.prompt).toBe("Production prompt");
expect(result.labels).toContain("production");
});
it("should fetch prompt by name and specific label", async () => {
const { context, projectId } = await createMcpTestSetup();
const promptName = `test-prompt-${nanoid()}`;
// Create v1 with staging label
await createPromptInDb({
name: promptName,
prompt: "Version 1",
projectId,
labels: ["staging"],
version: 1,
});
// Create v2 with production label
await createPromptInDb({
name: promptName,
prompt: "Version 2",
projectId,
labels: ["production"],
version: 2,
});
const result = (await handleGetPrompt(
{ name: promptName, label: "staging" },
context,
)) as { version: number; prompt: string };
expect(result.version).toBe(1);
expect(result.prompt).toBe("Version 1");
});
it("should fetch prompt by name and specific version", async () => {
const { context, projectId } = await createMcpTestSetup();
const promptName = `test-prompt-${nanoid()}`;
await createPromptInDb({
name: promptName,
prompt: "Version 1",
projectId,
version: 1,
});
await createPromptInDb({
name: promptName,
prompt: "Version 2",
projectId,
version: 2,
});
const result = (await handleGetPrompt(
{ name: promptName, version: 2 },
context,
)) as { version: number; prompt: string };
expect(result.version).toBe(2);
expect(result.prompt).toBe("Version 2");
});
it("should throw error when both label and version are specified", async () => {
const { context, projectId } = await createMcpTestSetup();
const promptName = `test-prompt-${nanoid()}`;
await createPromptInDb({
name: promptName,
prompt: "Test",
projectId,
});
// The input schema refinement should reject this
await expect(
handleGetPrompt(
{ name: promptName, label: "production", version: 1 },
context,
),
).rejects.toThrow();
});
it("should return error for non-existent prompt", async () => {
const { context } = await createMcpTestSetup();
await expect(
handleGetPrompt({ name: "non-existent-prompt" }, context),
).rejects.toThrow(/not found/i);
});
it("should return error for non-existent label", async () => {
const { context, projectId } = await createMcpTestSetup();
const promptName = `test-prompt-${nanoid()}`;
await createPromptInDb({
name: promptName,
prompt: "Test",
projectId,
labels: ["staging"],
});
await expect(
handleGetPrompt({ name: promptName, label: "production" }, context),
).rejects.toThrow(/not found/i);
});
it("should return error for non-existent version", async () => {
const { context, projectId } = await createMcpTestSetup();
const promptName = `test-prompt-${nanoid()}`;
await createPromptInDb({
name: promptName,
prompt: "Test",
projectId,
version: 1,
});
await expect(
handleGetPrompt({ name: promptName, version: 999 }, context),
).rejects.toThrow(/not found/i);
});
it("should use context.projectId for tenant isolation", async () => {
const { context: context1, projectId: projectId1 } =
await createMcpTestSetup();
const { context: context2, projectId: projectId2 } =
await createMcpTestSetup();
const promptName = `shared-name-${nanoid()}`;
// Create same-named prompt in both projects
await createPromptInDb({
name: promptName,
prompt: "Project 1 content",
projectId: projectId1,
labels: ["production", "latest"],
});
await createPromptInDb({
name: promptName,
prompt: "Project 2 content",
projectId: projectId2,
labels: ["production", "latest"],
});
// Each context should only see its own project's prompt
const result1 = (await handleGetPrompt(
{ name: promptName },
context1,
)) as { prompt: string };
expect(result1.prompt).toBe("Project 1 content");
const result2 = (await handleGetPrompt(
{ name: promptName },
context2,
)) as { prompt: string };
expect(result2.prompt).toBe("Project 2 content");
});
it("should handle special characters in prompt name", async () => {
const { context, projectId } = await createMcpTestSetup();
const promptName = `test-prompt-special!@#$%${nanoid()}`;
await createPromptInDb({
name: promptName,
prompt: "Special chars test",
projectId,
labels: ["production", "latest"],
});
const result = (await handleGetPrompt({ name: promptName }, context)) as {
name: string;
};
expect(result.name).toBe(promptName);
});
it("should include prompt config in response", async () => {
const { context, projectId } = await createMcpTestSetup();
const promptName = `test-prompt-${nanoid()}`;
await createPromptInDb({
name: promptName,
prompt: "Test",
projectId,
labels: ["production", "latest"],
config: { model: "gpt-4", temperature: 0.7 },
});
const result = (await handleGetPrompt({ name: promptName }, context)) as {
config: Record<string, unknown>;
};
expect(result.config).toEqual({ model: "gpt-4", temperature: 0.7 });
});
it("should include tags in response", async () => {
const { context, projectId } = await createMcpTestSetup();
const promptName = `test-prompt-${nanoid()}`;
await createPromptInDb({
name: promptName,
prompt: "Test",
projectId,
labels: ["production", "latest"],
tags: ["experimental", "v2"],
});
const result = (await handleGetPrompt({ name: promptName }, context)) as {
tags: string[];
};
expect(result.tags).toEqual(["experimental", "v2"]);
});
});
describe("listPrompts tool", () => {
it("should have readOnlyHint annotation", () => {
verifyToolAnnotations(listPromptsTool, { readOnlyHint: true });
});
it("should be available to in-app agent keys", async () => {
const context = mockServerContext({
inAppAgent: { permissions: "read" },
});
await expect(
toolRegistry.getEnabledTool(listPromptsTool.name, context),
).resolves.toBeDefined();
});
it("should list all prompts for project", async () => {
const { context, projectId } = await createMcpTestSetup();
// Create multiple prompts
const prompt1Name = `list-test-1-${nanoid()}`;
const prompt2Name = `list-test-2-${nanoid()}`;
await createPromptInDb({
name: prompt1Name,
prompt: "First prompt",
projectId,
});
await createPromptInDb({
name: prompt2Name,
prompt: "Second prompt",
projectId,
});
const result = (await handleListPrompts(
{ page: 1, limit: 100 },
context,
)) as {
data: Array<{ name: string }>;
meta: { totalItems: number };
};
// Should include our prompts (may include others from setup)
const names = result.data.map((p) => p.name);
expect(names).toContain(prompt1Name);
expect(names).toContain(prompt2Name);
expect(result.meta.totalItems).toBeGreaterThanOrEqual(2);
expect(Object.keys(result).sort()).toEqual(["data", "meta"]);
});
it("should filter by name", async () => {
const { context, projectId } = await createMcpTestSetup();
const uniquePrefix = `filter-name-${nanoid()}`;
await createPromptInDb({
name: `${uniquePrefix}-match`,
prompt: "Match",
projectId,
});
await createPromptInDb({
name: `other-${nanoid()}`,
prompt: "No match",
projectId,
});
const result = (await handleListPrompts(
{ name: `${uniquePrefix}-match`, page: 1, limit: 100 },
context,
)) as {
data: Array<{ name: string }>;
};
expect(result.data.length).toBe(1);
expect(result.data[0].name).toBe(`${uniquePrefix}-match`);
});
it("should filter by label", async () => {
const { context, projectId } = await createMcpTestSetup();
const promptName = `filter-label-${nanoid()}`;
await createPromptInDb({
name: promptName,
prompt: "Production version",
projectId,
labels: ["production"],
version: 1,
});
await createPromptInDb({
name: `other-${nanoid()}`,
prompt: "Staging version",
projectId,
labels: ["staging"],
version: 1,
});
const result = (await handleListPrompts(
{ label: "production", page: 1, limit: 100 },
context,
)) as {
data: Array<{ name: string; labels: string[] }>;
};
// All returned prompts should have production label
for (const prompt of result.data) {
expect(prompt.labels).toContain("production");
}
});
it("should filter by tag", async () => {
const { context, projectId } = await createMcpTestSetup();
const promptName = `filter-tag-${nanoid()}`;
await createPromptInDb({
name: promptName,
prompt: "Tagged prompt",
projectId,
tags: ["experimental"],
});
await createPromptInDb({
name: `untagged-${nanoid()}`,
prompt: "Untagged prompt",
projectId,
tags: [],
});
const result = (await handleListPrompts(
{ tag: "experimental", page: 1, limit: 100 },
context,
)) as {
data: Array<{ name: string; tags: string[] }>;
};
// Should only return prompts with experimental tag
expect(result.data.length).toBeGreaterThan(0);
for (const prompt of result.data) {
expect(prompt.tags).toContain("experimental");
}
});
it("should filter by fromUpdatedAt", async () => {
const { context, projectId } = await createMcpTestSetup();
const oldPrompt = `filter-from-updated-${nanoid()}`;
const newPrompt = `filter-from-updated-${nanoid()}`;
const oldDate = new Date("2026-01-01T00:00:00.000Z");
const newDate = new Date("2026-02-01T00:00:00.000Z");
await prisma.prompt.create({
data: {
name: oldPrompt,
prompt: "old",
labels: [],
tags: [],
type: "text",
version: 1,
config: {},
createdBy: "test-user",
createdAt: oldDate,
updatedAt: oldDate,
project: { connect: { id: projectId } },
},
});
await prisma.prompt.create({
data: {
name: newPrompt,
prompt: "new",
labels: [],
tags: [],
type: "text",
version: 1,
config: {},
createdBy: "test-user",
createdAt: newDate,
updatedAt: newDate,
project: { connect: { id: projectId } },
},
});
const result = (await handleListPrompts(
{
fromUpdatedAt: "2026-01-15T00:00:00.000Z",
page: 1,
limit: 100,
},
context,
)) as { data: Array<{ name: string }> };
const names = result.data.map((p) => p.name);
expect(names).toContain(newPrompt);
expect(names).not.toContain(oldPrompt);
});
it("should filter by toUpdatedAt", async () => {
const { context, projectId } = await createMcpTestSetup();
const oldPrompt = `filter-to-updated-${nanoid()}`;
const newPrompt = `filter-to-updated-${nanoid()}`;
const oldDate = new Date("2026-01-01T00:00:00.000Z");
const newDate = new Date("2026-02-01T00:00:00.000Z");
await prisma.prompt.create({
data: {
name: oldPrompt,
prompt: "old",
labels: [],
tags: [],
type: "text",
version: 1,
config: {},
createdBy: "test-user",
createdAt: oldDate,
updatedAt: oldDate,
project: { connect: { id: projectId } },
},
});
await prisma.prompt.create({
data: {
name: newPrompt,
prompt: "new",
labels: [],
tags: [],
type: "text",
version: 1,
config: {},
createdBy: "test-user",
createdAt: newDate,
updatedAt: newDate,
project: { connect: { id: projectId } },
},
});
const result = (await handleListPrompts(
{
toUpdatedAt: "2026-01-15T00:00:00.000Z",
page: 1,
limit: 100,
},
context,
)) as { data: Array<{ name: string }> };
const names = result.data.map((p) => p.name);
expect(names).toContain(oldPrompt);
expect(names).not.toContain(newPrompt);
});
it("should return error when fromUpdatedAt is after toUpdatedAt", async () => {
const { context } = await createMcpTestSetup();
await expect(
handleListPrompts(
{
fromUpdatedAt: "2026-02-02T00:00:00.000Z",
toUpdatedAt: "2026-02-01T00:00:00.000Z",
page: 1,
limit: 50,
},
context,
),
).rejects.toThrow(/fromUpdatedAt.*<=.*toUpdatedAt/i);
});
it("should handle pagination with page and limit", async () => {
const { context, projectId } = await createMcpTestSetup();
// Create enough prompts to test pagination
for (let i = 0; i < 5; i++) {
await createPromptInDb({
name: `pagination-test-${i}-${nanoid()}`,
prompt: `Prompt ${i}`,
projectId,
});
}
const result = (await handleListPrompts(
{ page: 1, limit: 2 },
context,
)) as {
data: Array<{ name: string }>;
meta: { page: number; limit: number; totalPages: number };
};
expect(result.data.length).toBeLessThanOrEqual(2);
expect(result.meta.page).toBe(1);
expect(result.meta.limit).toBe(2);
expect(result.meta.totalPages).toBeGreaterThanOrEqual(1);
});
it("should return empty results for no matches", async () => {
const { context } = await createMcpTestSetup();
const result = (await handleListPrompts(
{ name: `non-existent-${nanoid()}`, page: 1, limit: 100 },
context,
)) as {
data: Array<unknown>;
meta: { totalItems: number };
};
expect(result.data).toEqual([]);
expect(result.meta.totalItems).toBe(0);
});
it("should use context.projectId for tenant isolation", async () => {
const { context: context1, projectId: projectId1 } =
await createMcpTestSetup();
const { context: context2 } = await createMcpTestSetup();
const uniqueName = `isolation-test-${nanoid()}`;
// Create prompt only in project 1
await createPromptInDb({
name: uniqueName,
prompt: "Project 1 only",
projectId: projectId1,
});
// Project 1 should see it
const result1 = (await handleListPrompts(
{ name: uniqueName, page: 1, limit: 100 },
context1,
)) as { data: Array<unknown> };
expect(result1.data.length).toBe(1);
// Project 2 should not see it
const result2 = (await handleListPrompts(
{ name: uniqueName, page: 1, limit: 100 },
context2,
)) as { data: Array<unknown> };
expect(result2.data.length).toBe(0);
});
it("should respect default pagination values", async () => {
const { context } = await createMcpTestSetup();
const result = (await handleListPrompts(
{ page: 1, limit: 100 },
context,
)) as {
meta: { page: number; limit: number };
};
// Default values from validation schema
expect(result.meta.page).toBe(1);
expect(result.meta.limit).toBeLessThanOrEqual(100); // Max limit
});
it("should include prompt metadata in list results", async () => {
const { context, projectId } = await createMcpTestSetup();
const promptName = `metadata-test-${nanoid()}`;
await createPromptInDb({
name: promptName,
prompt: "Test",
projectId,
labels: ["production"],
tags: ["important"],
version: 1,
});
const result = (await handleListPrompts(
{ name: promptName, page: 1, limit: 100 },
context,
)) as {
data: Array<{
name: string;
version: number;
labels: string[];
tags: string[];
}>;
};
expect(result.data[0].name).toBe(promptName);
expect(result.data[0].labels).toContain("production");
expect(result.data[0].tags).toContain("important");
});
});
describe("getPromptUnresolved tool", () => {
it("should have readOnlyHint annotation", () => {
verifyToolAnnotations(getPromptUnresolvedTool, { readOnlyHint: true });
});
it("should be available to in-app agent keys", async () => {
const context = mockServerContext({
inAppAgent: { permissions: "read" },
});
await expect(
toolRegistry.getEnabledTool(getPromptUnresolvedTool.name, context),
).resolves.toBeDefined();
});
it("should fetch latest prompt without resolving dependencies by default", async () => {
const { context, projectId } = await createMcpTestSetup();
const promptName = `test-prompt-unresolved-${nanoid()}`;
const rawPromptContent =
"You are a helpful assistant. @@@langfusePrompt:name=base-instructions|label=production@@@";
await createPromptInDb({
name: promptName,
prompt: "Production prompt",
projectId,
labels: ["production"],
version: 1,
});
await createPromptInDb({
name: promptName,
prompt: rawPromptContent,
projectId,
labels: ["latest"],
version: 2,
});
const result = (await handleGetPromptUnresolved(
{ name: promptName },
context,
)) as {
name: string;
version: number;
prompt: string;
labels: string[];
};
expect(result.name).toBe(promptName);
expect(result.version).toBe(2);
expect(result.labels).toContain("latest");
// Verify dependency tags are NOT resolved
expect(result.prompt).toBe(rawPromptContent);
expect(result.prompt).toContain(
"@@@langfusePrompt:name=base-instructions|label=production@@@",
);
});
it("should fetch prompt by name and specific label without resolution", async () => {
const { context, projectId } = await createMcpTestSetup();
const promptName = `test-prompt-unresolved-${nanoid()}`;
// Create v1 with staging label
await createPromptInDb({
name: promptName,
prompt: "Version 1 @@@langfusePrompt:name=helper|label=staging@@@",
projectId,
labels: ["staging"],
version: 1,
});
// Create v2 with production label
await createPromptInDb({
name: promptName,
prompt: "Version 2 @@@langfusePrompt:name=helper|label=production@@@",
projectId,
labels: ["production"],
version: 2,
});
const result = (await handleGetPromptUnresolved(
{ name: promptName, label: "staging" },
context,
)) as { version: number; prompt: string };
expect(result.version).toBe(1);
expect(result.prompt).toBe(
"Version 1 @@@langfusePrompt:name=helper|label=staging@@@",
);
});
it("should fetch prompt by name and specific version without resolution", async () => {
const { context, projectId } = await createMcpTestSetup();
const promptName = `test-prompt-unresolved-${nanoid()}`;
await createPromptInDb({
name: promptName,
prompt: "V1 content @@@langfusePrompt:name=dep|label=v1@@@",
projectId,
labels: ["staging"],
version: 1,
});
await createPromptInDb({
name: promptName,
prompt: "V2 content @@@langfusePrompt:name=dep|label=v2@@@",
projectId,
labels: ["production"],
version: 2,
});
const result = (await handleGetPromptUnresolved(
{ name: promptName, version: 1 },
context,
)) as { version: number; prompt: string };
expect(result.version).toBe(1);
expect(result.prompt).toBe(
"V1 content @@@langfusePrompt:name=dep|label=v1@@@",
);
});
it("should throw error if prompt not found", async () => {
const { context } = await createMcpTestSetup();
await expect(
handleGetPromptUnresolved(
{ name: "non-existent-prompt-12345" },
context,
),
).rejects.toThrow("Prompt 'non-existent-prompt-12345' not found");
});
it("should throw error when both label and version are specified", async () => {
const { context } = await createMcpTestSetup();
await expect(
handleGetPromptUnresolved(
{ name: "test", label: "production", version: 1 },
context,
),
).rejects.toThrow(
"Cannot specify both label and version - they are mutually exclusive",
);
});
it("should return raw chat prompt without resolving dependencies", async () => {
const { context, projectId } = await createMcpTestSetup();
const promptName = `test-chat-unresolved-${nanoid()}`;
const chatMessages = [
{
role: "system",
content:
"You are helpful @@@langfusePrompt:name=system-base|label=production@@@",
},
{
role: "user",
content: "@@@langfusePrompt:name=user-template|label=production@@@",
},
];
await createPromptInDb({
name: promptName,
prompt: chatMessages,
projectId,
labels: ["production", "latest"],
version: 1,
type: "chat",
});
const result = (await handleGetPromptUnresolved(
{ name: promptName },
context,
)) as {
name: string;
type: string;
prompt: Array<{ role: string; content: string }>;
};
expect(result.type).toBe("chat");
expect(result.prompt).toEqual(chatMessages);
expect(result.prompt[0].content).toContain(
"@@@langfusePrompt:name=system-base|label=production@@@",
);
});
});
describe("dashboard read tools", () => {
const dashboardReadTools = [
listDashboardsTool,
getDashboardTool,
listDashboardWidgetsTool,
getDashboardWidgetTool,
];
it("should have readOnlyHint annotations", () => {
for (const tool of dashboardReadTools) {
verifyToolAnnotations(tool, { readOnlyHint: true });
}
});
it("should list and get dashboards and widgets with product URLs", async () => {
const setup = await createMcpTestSetup();
const dashboard = (await handleCreateDashboard(
{ name: `mcp-dashboard-${nanoid()}`, description: "" },
setup.context,
)) as { id: string };
const widget = (await handleCreateDashboardWidget(
{
name: `mcp-widget-${nanoid()}`,
description: "Created by MCP",
view: "observations",
dimensions: [],
metrics: [{ measure: "count", agg: "count" }],
filters: [],
chartType: "NUMBER",
chartConfig: { type: "NUMBER" },
},
setup.context,
)) as { id: string };
const listedDashboards = (await handleListDashboards(
{ page: 1, limit: 50 },
setup.context,
)) as { data: Array<{ id: string; url: string }> };
expect(listedDashboards.data.map((item) => item.id)).toContain(
dashboard.id,
);
const fetchedDashboard = (await handleGetDashboard(
{ dashboardId: dashboard.id },
setup.context,
)) as { id: string; url: string };
expect(fetchedDashboard).toMatchObject({
id: dashboard.id,
url: buildDashboardUrl({
projectId: setup.projectId,
dashboardId: dashboard.id,
}),
});
const listedWidgets = (await handleListDashboardWidgets(
{ page: 1, limit: 50 },
setup.context,
)) as { data: Array<{ id: string; url: string }> };
expect(listedWidgets.data.map((item) => item.id)).toContain(widget.id);
const fetchedWidget = (await handleGetDashboardWidget(
{ widgetId: widget.id },
setup.context,
)) as { id: string; url: string };
expect(fetchedWidget).toMatchObject({
id: widget.id,
url: buildDashboardWidgetUrl({
projectId: setup.projectId,
widgetId: widget.id,
}),
});
});
});
});