Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

8.2 KB
Raw Blame
/**
 * MCP Test Helpers
 *
 * Shared utilities for testing MCP server tools.
 * These helpers allow direct testing of tool handlers without HTTP overhead.
 */

import { randomUUID } from "crypto";
import { prisma, type Role } from "@langfuse/shared/src/db";
import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";
import { createApiKey } from "@langfuse/shared/src/server/auth/apiKeys";
import {
  OrganizationId,
  ProjectId,
  SystemRoleId,
  UserId,
} from "@langfuse/shared/rbac";
import type { ServerContext } from "@/src/features/mcp/types";
import { authenticator } from "@/src/features/apiKey/authenticator";

/**
 * Creates a complete MCP test setup including:
 * - Organization
 * - Project
 * - API key
 * - ServerContext for MCP tool handlers
 */
export async function createMcpTestSetup(): Promise<{
  projectId: string;
  orgId: string;
  apiKeyId: string;
  auth: string;
  context: ServerContext;
}> {
  const result = await createOrgProjectAndApiKey();
  const { projectId, orgId, auth } = result;

  // Extract apiKeyId from created API key
  const apiKey = await prisma.apiKey.findFirst({
    where: { projectId, publicKey: result.publicKey },
    select: { id: true },
  });

  if (!apiKey) {
    throw new Error("Failed to create API key for test setup");
  }

  const authenticated = await authenticator.authenticate({
    headers: { authorization: auth },
  });
  if (!authenticated.success) throw authenticated.error;

  const context: ServerContext = {
    projectId,
    orgId,
    apiKeyId: apiKey.id,
    accessLevel: "project",
    publicKey: result.publicKey,
    plan: "oss",
    rateLimitOverrides: [],
    accessAuditLogEnabled: false,
    auth: authenticated.context,
  };

  return {
    projectId,
    orgId,
    apiKeyId: apiKey.id,
    auth,
    context,
  };
}

/**
 * Creates a user with an org membership of the given role. Project role is
 * inherited from the org membership unless a project override is added later.
 */
export async function createUserWithOrgRole(params: {
  orgId: string;
  role: Role;
}): Promise<{ userId: string }> {
  const user = await prisma.user.create({
    data: {
      email: `mcp-rbac-${randomUUID()}@langfuse.com`,
      name: `MCP ${params.role} user`,
    },
  });

  await prisma.organizationMembership.create({
    data: {
      orgId: params.orgId,
      userId: user.id,
      role: params.role,
    },
  });

  return { userId: user.id };
}

/**
 * Mints a project API key with isInAppAgentKey + createdByUserId, matching
 * the temporary keys the in-app agent runtime issues for MCP.
 */
export async function createInAppAgentMcpContext(params: {
  projectId: string;
  orgId: string;
  createdByUserId?: string;
}): Promise<{ apiKeyId: string; context: ServerContext }> {
  const creatorUserId =
    params.createdByUserId ??
    (
      await prisma.user.create({
        data: { email: `apikey-creator-${randomUUID()}@example.com` },
      })
    ).id;
  const apiKey = await createApiKey(prisma, {
    tenant: OrganizationId(params.orgId),
    scope: ProjectId(params.projectId),
    role: SystemRoleId("LEGACY_PROJECT_API_KEY"),
    createdBy: UserId(creatorUserId),
    name: "In-app agent MCP session",
    isInAppAgentKey: true,
  });
  const authenticated = await authenticator.authenticate({
    headers: {
      authorization: `Basic ${Buffer.from(`${apiKey.publicKey}:${apiKey.secretKey}`).toString("base64")}`,
    },
    allowInAppAgentKey: true,
  });
  if (!authenticated.success) throw authenticated.error;

  return {
    apiKeyId: apiKey.id,
    context: {
      projectId: params.projectId,
      orgId: params.orgId,
      apiKeyId: apiKey.id,
      accessLevel: "project",
      publicKey: apiKey.publicKey,
      plan: "oss",
      rateLimitOverrides: [],
      accessAuditLogEnabled: false,
      auth: authenticated.context,
    },
  };
}

export async function protectPromptLabel(params: {
  projectId: string;
  label: string;
}): Promise<void> {
  await prisma.promptProtectedLabels.create({
    data: {
      projectId: params.projectId,
      label: params.label,
    },
  });
}

/**
 * Creates a mock ServerContext for testing.
 * Use this when you need a context but don't want to create actual DB records.
 */
export function mockServerContext(
  overrides?: Partial<ServerContext>,
): ServerContext {
  return {
    projectId: overrides?.projectId ?? "test-project-id",
    orgId: overrides?.orgId ?? "test-org-id",
    apiKeyId: overrides?.apiKeyId ?? "test-api-key-id",
    accessLevel: "project",
    publicKey: overrides?.publicKey ?? "pk-lf-test",
    plan: overrides?.plan ?? "oss",
    rateLimitOverrides: overrides?.rateLimitOverrides ?? [],
    accessAuditLogEnabled: overrides?.accessAuditLogEnabled ?? false,
    ...overrides,
  };
}

export const mcpEvalOutputDefinition = {
  dataType: "NUMERIC" as const,
  reasoning: { description: "Why the score was assigned" },
  score: { description: "A score between 0 and 1" },
};

/**
 * Verifies that an audit log entry was created for an MCP operation.
 * Returns the audit log entry for further assertions.
 */
export async function verifyAuditLog(params: {
  projectId: string;
  resourceType: string;
  resourceId?: string;
  action: "create" | "update" | "delete";
  apiKeyId: string;
}): Promise<{
  id: string;
  action: string;
  resourceType: string;
  resourceId: string;
  before: unknown;
  after: unknown;
}> {
  const auditLogs = await prisma.auditLog.findMany({
    where: {
      projectId: params.projectId,
      resourceType: params.resourceType,
      action: params.action,
      apiKeyId: params.apiKeyId,
      ...(params.resourceId && { resourceId: params.resourceId }),
    },
    orderBy: {
      createdAt: "desc",
    },
    take: 1,
  });

  if (auditLogs.length === 0) {
    throw new Error(
      `No audit log found for ${params.action} ${params.resourceType} in project ${params.projectId}`,
    );
  }

  const log = auditLogs[0];

  return {
    id: log.id,
    action: log.action,
    resourceType: log.resourceType,
    resourceId: log.resourceId,
    before: log.before,
    after: log.after,
  };
}

/**
 * Creates a prompt directly in the database for testing.
 * Similar to the helper in prompts.v2.servertest.ts.
 */
export async function createPromptInDb(params: {
  name: string;
  prompt: string | unknown; // Can be string (text) or array (chat messages)
  projectId: string;
  labels?: string[];
  version?: number;
  config?: Record<string, unknown>;
  tags?: string[];
  type?: "text" | "chat";
  createdBy?: string;
}) {
  return await prisma.prompt.create({
    data: {
      name: params.name,

      prompt: params.prompt as any, // Prisma's JsonValue type - can be string or object
      labels: params.labels ?? [],
      version: params.version ?? 1,

      config: (params.config ?? {}) as any, // Prisma's JsonValue type - safe because we control the input
      tags: params.tags ?? [],
      type: params.type ?? "text",
      createdBy: params.createdBy ?? "test-user",
      project: {
        connect: { id: params.projectId },
      },
    },
  });
}

/**
 * Helper to check if a tool has the correct MCP annotations.
 */
export function verifyToolAnnotations(
  toolDefinition: { annotations?: Record<string, boolean> },
  expectedAnnotations: {
    readOnlyHint?: boolean;
    destructiveHint?: boolean;
    expensiveHint?: boolean;
  },
): void {
  const annotations = toolDefinition.annotations ?? {};

  if (expectedAnnotations.readOnlyHint !== undefined) {
    expect(annotations.readOnlyHint).toBe(expectedAnnotations.readOnlyHint);
  }

  if (expectedAnnotations.destructiveHint !== undefined) {
    expect(annotations.destructiveHint).toBe(
      expectedAnnotations.destructiveHint,
    );
  }

  if (expectedAnnotations.expensiveHint !== undefined) {
    expect(annotations.expensiveHint).toBe(expectedAnnotations.expensiveHint);
  }
}

/**
 * Waits for a condition to be true with timeout.
 * Useful for async operations that need eventual consistency.
 */
export async function waitFor(
  condition: () => Promise<boolean>,
  timeoutMs = 5000,
  intervalMs = 100,
): Promise<void> {
  const startTime = Date.now();

  while (Date.now() - startTime < timeoutMs) {
    if (await condition()) {
      return;
    }
    await new Promise((resolve) => setTimeout(resolve, intervalMs));
  }

  throw new Error(`Condition not met within ${timeoutMs}ms`);
}