Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

16.3 KB
Raw Blame
import { readdirSync } from "node:fs";
import path from "node:path";
import { randomUUID } from "node:crypto";

import type { NextApiRequest, NextApiResponse } from "next";
import { createMocks } from "node-mocks-http";

import { env } from "@/src/env.mjs";
import { prisma } from "@langfuse/shared/src/db";
import {
  createApiKey,
  createBasicAuthHeader,
  createOrgProjectAndApiKey,
} from "@langfuse/shared/src/server";
import {
  OrganizationId,
  ProjectId,
  SystemRoleId,
  UserId,
} from "@langfuse/shared/rbac";

// Pins the public-API authorization seam to legacy: sweeps every route across
// migration modes and key kinds, recording each cell's status. Shadow and
// enforce must equal legacy, save documented enforce divergences; a snapshot
// pins legacy itself. Value is status only.

type HttpMethod = "GET" | "POST" | "PUT" | "PATCH" | "DELETE";

type Route = { route: string; methods: HttpMethod[] };

// Routes under test, grouped by domain. `route` is the path under
// pages/api/public/ (import path + completeness key); `methods` are its real
// supported methods. Project routes dispatch through createAuthedProjectAPIRoute.
const projectRoutes: Route[] = [
  { route: "traces/index", methods: ["GET", "POST", "DELETE"] },
  { route: "traces/[traceId]", methods: ["GET", "DELETE"] },
  { route: "observations/index", methods: ["GET"] },
  { route: "observations/[observationId]", methods: ["GET"] },
  { route: "v2/observations/index", methods: ["GET"] },
  { route: "sessions/index", methods: ["GET"] },
  { route: "sessions/[sessionId]", methods: ["GET"] },
  { route: "scores/index", methods: ["GET", "POST"] },
  { route: "scores/[scoreId]", methods: ["GET", "DELETE"] },
  { route: "v2/scores/index", methods: ["GET"] },
  { route: "v2/scores/[scoreId]", methods: ["GET"] },
  { route: "v3/scores/index", methods: ["GET"] },
  { route: "score-configs/index", methods: ["GET", "POST"] },
  { route: "score-configs/[configId]", methods: ["GET", "PATCH"] },
  { route: "comments/index", methods: ["GET", "POST"] },
  { route: "comments/[commentId]", methods: ["GET"] },
  { route: "annotation-queues/index", methods: ["GET", "POST"] },
  { route: "annotation-queues/[queueId]/index", methods: ["GET"] },
  {
    route: "annotation-queues/[queueId]/assignments",
    methods: ["POST", "DELETE"],
  },
  {
    route: "annotation-queues/[queueId]/items/index",
    methods: ["GET", "POST"],
  },
  {
    route: "annotation-queues/[queueId]/items/[itemId]",
    methods: ["GET", "PATCH", "DELETE"],
  },
  { route: "datasets/index", methods: ["GET", "POST"] },
  { route: "datasets/[name]/index", methods: ["GET"] },
  { route: "datasets/[name]/runs/index", methods: ["GET"] },
  { route: "datasets/[name]/runs/[runName]", methods: ["GET", "DELETE"] },
  { route: "v2/datasets/index", methods: ["GET", "POST"] },
  { route: "v2/datasets/[datasetName]/index", methods: ["GET"] },
  { route: "dataset-items/index", methods: ["GET", "POST"] },
  { route: "dataset-items/[datasetItemId]", methods: ["GET", "DELETE"] },
  { route: "dataset-run-items", methods: ["GET", "POST"] },
  { route: "experiments/index", methods: ["GET"] },
  { route: "experiment-items/index", methods: ["GET"] },
  { route: "models/index", methods: ["GET", "POST"] },
  { route: "models/[modelId]/index", methods: ["GET", "PUT", "DELETE"] },
  { route: "media/index", methods: ["POST"] },
  { route: "media/[mediaId]", methods: ["GET", "PATCH"] },
  { route: "metrics/index", methods: ["GET"] },
  { route: "metrics/daily", methods: ["GET"] },
  { route: "v2/metrics", methods: ["GET"] },
  { route: "events", methods: ["POST"] },
  { route: "feedback", methods: ["POST"] },
  { route: "generations", methods: ["POST", "PATCH"] },
  { route: "spans", methods: ["POST", "PATCH"] },
  { route: "llm-connections/index", methods: ["GET", "PUT"] },
  { route: "llm-connections/[id]", methods: ["DELETE"] },
  { route: "unstable/dashboards/index", methods: ["GET", "POST"] },
  {
    route: "unstable/dashboards/[dashboardId]",
    methods: ["GET", "PATCH", "DELETE"],
  },
  {
    route: "unstable/dashboards/[dashboardId]/placements/index",
    methods: ["POST"],
  },
  {
    route: "unstable/dashboards/[dashboardId]/placements/[placementId]",
    methods: ["PATCH", "DELETE"],
  },
  { route: "unstable/dashboard-widgets/index", methods: ["GET", "POST"] },
  {
    route: "unstable/dashboard-widgets/[widgetId]",
    methods: ["GET", "PATCH", "DELETE"],
  },
  { route: "v2/evaluators/index", methods: ["GET", "POST"] },
  {
    route: "v2/evaluators/[evaluatorId]",
    methods: ["GET", "PATCH", "DELETE"],
  },
  { route: "v2/evaluators/[evaluatorId]/versions", methods: ["GET"] },
  { route: "v2/evaluation-rules/index", methods: ["GET", "POST"] },
  {
    route: "v2/evaluation-rules/[evaluationRuleId]",
    methods: ["GET", "PATCH", "DELETE"],
  },
  { route: "ingestion", methods: ["POST"] },
  { route: "otel/v1/traces/index", methods: ["POST"] },
  { route: "otel/v1/metrics/index", methods: ["POST"] },
  { route: "prompts", methods: ["GET", "POST"] },
  { route: "v2/prompts/index", methods: ["GET", "POST"] },
  { route: "v2/prompts/[promptName]/index", methods: ["GET", "DELETE"] },
  {
    route: "v2/prompts/[promptName]/versions/[promptVersion]",
    methods: ["PATCH"],
  },
  { route: "skills/index", methods: ["GET", "POST"] },
  { route: "skills/[skillName]/index", methods: ["GET", "PATCH"] },
  {
    route: "skills/[skillName]/versions/[skillVersion]",
    methods: ["PATCH", "DELETE"],
  },
  { route: "skills/files/content", methods: ["GET"] },
  { route: "unstable/skills/index", methods: ["GET", "POST"] },
  { route: "unstable/skills/[skillName]/index", methods: ["GET", "PATCH"] },
  {
    route: "unstable/skills/[skillName]/versions/[skillVersion]",
    methods: ["PATCH", "DELETE"],
  },
  { route: "unstable/skills/files/content", methods: ["GET"] },
  { route: "unstable/topics/index", methods: ["GET"] },
  { route: "unstable/topics/[topicId]", methods: ["GET"] },
  { route: "unstable/topic-facets/index", methods: ["GET"] },
  { route: "unstable/topic-facets/[facetId]", methods: ["GET"] },
];

// Org and misc routes call shadowAuth directly from the handler body.
const orgRoutes: Route[] = [
  { route: "organizations/apiKeys/index", methods: ["GET"] },
  {
    route: "organizations/llm-connections/index",
    methods: ["GET", "PUT"],
  },
  {
    route: "organizations/llm-connections/[id]",
    methods: ["DELETE"],
  },
  {
    route: "organizations/memberships/index",
    methods: ["GET", "PUT", "DELETE"],
  },
  { route: "organizations/projects/index", methods: ["GET"] },
  { route: "projects/index", methods: ["GET", "POST"] },
  { route: "projects/[projectId]/index", methods: ["PUT", "DELETE"] },
  { route: "projects/[projectId]/apiKeys/index", methods: ["GET", "POST"] },
  { route: "projects/[projectId]/apiKeys/[apiKeyId]", methods: ["DELETE"] },
  {
    route: "projects/[projectId]/memberships/index",
    methods: ["GET", "PUT", "DELETE"],
  },
  { route: "scim/ResourceTypes", methods: ["GET"] },
  { route: "scim/Schemas", methods: ["GET"] },
  { route: "scim/ServiceProviderConfig", methods: ["GET"] },
  { route: "scim/Users/index", methods: ["GET", "POST"] },
  { route: "scim/Users/[id]", methods: ["GET", "PUT", "PATCH", "DELETE"] },
  { route: "integrations/blob-storage/index", methods: ["GET", "PUT"] },
  { route: "integrations/blob-storage/[id]", methods: ["GET", "DELETE"] },
];

// Public files outside both tables, subtracted from the filesystem walk before
// the completeness guard compares it to the tables.
const denylistPrefixes = [
  "health", // liveness probe
  "ready", // readiness probe
  "mcp", // MCP server, own auth path
  "otel/otlp-proto", // generated protobuf, not a route
  "slack", // Slack OAuth, own auth path
];

type ApiKeyKind = "org" | "project" | "agent" | "admin";
const apiKeyKinds: ApiKeyKind[] = ["org", "project", "agent", "admin"];

type MigrationMode = "legacy" | "shadow" | "enforce";

const adminApiKey = "test-admin-api-key-project-route-parity";
// Valid-shaped id that does not exist, so a good key lands past auth (404/400).
const nonexistentId = "00000000-0000-0000-0000-000000000000";

type KeyPair = { publicKey: string; secretKey: string };

const keys: Record<Exclude<ApiKeyKind, "admin">, KeyPair> = {
  org: { publicKey: "", secretKey: "" },
  project: { publicKey: "", secretKey: "" },
  agent: { publicKey: "", secretKey: "" },
};
let fixtureProjectId = "";

let originalMigration: string | undefined;
let originalAdminApiKey: string | undefined;
let matrices: Record<MigrationMode, Record<string, number>>;

/** queryForRoute maps each [param] path segment to a fixed nonexistent id, as Next populates req.query. */
function queryForRoute(route: string): Record<string, string> {
  const query: Record<string, string> = {};
  for (const match of route.matchAll(/\[([^\]]+)\]/g)) {
    query[match[1]!] = nonexistentId;
  }
  if (
    route === "skills/files/content" ||
    route === "unstable/skills/files/content"
  ) {
    query.sha256Hashes = Buffer.alloc(32).toString("base64");
  }
  return query;
}

/** getHeaders derives a key kind's header sets: org/project/agent send basic + bearer, admin sends its single triple. */
function getHeaders(
  apiKeyKind: ApiKeyKind,
  route: string,
): { headerKind: string; headers: Record<string, string> }[] {
  if (apiKeyKind === "admin") {
    return [
      {
        headerKind: "admin",
        headers: {
          authorization: `Bearer ${adminApiKey}`,
          "x-langfuse-admin-api-key": adminApiKey,
          "x-langfuse-project-id": fixtureProjectId,
        },
      },
    ];
  }
  const { publicKey, secretKey } = keys[apiKeyKind];
  // An org key has no bound project, so a realistic org request names its target
  // via header unless the URL already does; project/agent keys resolve their
  // project from the key itself.
  const target =
    apiKeyKind === "org" && !route.includes("[projectId]")
      ? { "x-langfuse-project-id": fixtureProjectId }
      : undefined;
  return [
    {
      headerKind: "basic",
      headers: {
        authorization: createBasicAuthHeader(publicKey, secretKey),
        ...target,
      },
    },
    {
      headerKind: "bearer",
      headers: { authorization: `Bearer ${publicKey}`, ...target },
    },
  ];
}

/** callRoute imports the route by convention and invokes its handler with a mocked request, returning the status code. */
async function callRoute(
  route: string,
  method: HttpMethod,
  headers: Record<string, string>,
): Promise<number> {
  const mod = await import(
    /* @vite-ignore */ `@/src/pages/api/public/${route}`
  );
  const handler = mod.default as (
    req: NextApiRequest,
    res: NextApiResponse,
  ) => Promise<void>;
  const { req, res } = createMocks<NextApiRequest, NextApiResponse>({
    method,
    query: queryForRoute(route),
    body: method === "GET" ? undefined : {},
    headers,
  });
  await handler(req, res);
  return res._getStatusCode();
}

type Cell = { key: string; run: () => Promise<number> };

// node-mocks-http can't supply the raw request stream otel/v1/traces reads, so these cells hang; only the project key passes, identically in every mode.
const streamReadingCells = new Set([
  "POST otel/v1/traces/index | project/basic",
  "POST otel/v1/traces/index | project/bearer",
]);

/** matrixCells lists every route × method × key kind × header kind cell in source order. */
function matrixCells(routes: Route[]): Cell[] {
  const cells: Cell[] = [];
  for (const { route, methods } of routes) {
    for (const method of methods) {
      for (const apiKeyKind of apiKeyKinds) {
        for (const { headerKind, headers } of getHeaders(apiKeyKind, route)) {
          const key = `${method} ${route} | ${apiKeyKind}/${headerKind}`;
          if (streamReadingCells.has(key)) continue;
          cells.push({ key, run: () => callRoute(route, method, headers) });
        }
      }
    }
  }
  return cells;
}

/** runMatrix runs every cell in one mode with bounded concurrency, returning a flat status-by-cell map in source order. */
async function runMatrix(mode: MigrationMode): Promise<Record<string, number>> {
  (env as any).API_AUTH_MIGRATION = mode;
  const cells = matrixCells([...projectRoutes, ...orgRoutes]);
  const statuses = new Array<number>(cells.length);
  let next = 0;
  const worker = async () => {
    while (next < cells.length) {
      const index = next++;
      statuses[index] = await cells[index]!.run();
    }
  };
  await Promise.all(Array.from({ length: 16 }, worker));
  const result: Record<string, number> = {};
  cells.forEach((cell, index) => {
    result[cell.key] = statuses[index]!;
  });
  return result;
}

/** walkRoutes lists every route file under pages/api/public/ as a table-style route key. */
function walkRoutes(): string[] {
  const base = path.resolve(process.cwd(), "src/pages/api/public");
  const routes: string[] = [];
  const walk = (dir: string) => {
    for (const entry of readdirSync(dir, { withFileTypes: true })) {
      const full = path.join(dir, entry.name);
      if (entry.isDirectory()) {
        walk(full);
      } else if (entry.name.endsWith(".ts")) {
        routes.push(
          path
            .relative(base, full)
            .replace(/\.ts$/, "")
            .split(path.sep)
            .join("/"),
        );
      }
    }
  };
  walk(base);
  return routes;
}

// divergences lists cells whose status differs from the main baseline the
// snapshot records; `legacy`/`enforce` give that mode's actual status.
const divergences: Record<string, { legacy?: number; enforce?: number }> = {
  "GET projects/index | org/basic": { enforce: 200 },
  // ingestion had no tier gate on main (org key -> 401); shadowAuth adds one -> 403.
  "POST ingestion | org/basic": { legacy: 403, enforce: 400 },
};

describe("public-api auth parity", () => {
  beforeAll(async () => {
    originalMigration = (env as any).API_AUTH_MIGRATION;
    originalAdminApiKey = (env as any).ADMIN_API_KEY;
    (env as any).ADMIN_API_KEY = adminApiKey;

    const base = await createOrgProjectAndApiKey();
    fixtureProjectId = base.projectId;
    keys.project = { publicKey: base.publicKey, secretKey: base.secretKey };

    const orgKeyCreator = await prisma.user.create({
      data: { email: `apikey-creator-${randomUUID()}@example.com` },
    });
    const org = await createApiKey(prisma, {
      tenant: OrganizationId(base.orgId),
      scope: OrganizationId(base.orgId),
      role: SystemRoleId("LEGACY_ORGANIZATION_API_KEY"),
      createdBy: UserId(orgKeyCreator.id),
    });
    keys.org = { publicKey: org.publicKey, secretKey: org.secretKey };

    const agentKeyCreator = await prisma.user.create({
      data: { email: `apikey-creator-${randomUUID()}@example.com` },
    });
    const agent = await createApiKey(prisma, {
      tenant: OrganizationId(base.orgId),
      scope: ProjectId(base.projectId),
      role: SystemRoleId("LEGACY_PROJECT_API_KEY"),
      createdBy: UserId(agentKeyCreator.id),
      isInAppAgentKey: true,
    });
    keys.agent = { publicKey: agent.publicKey, secretKey: agent.secretKey };

    matrices = {
      legacy: await runMatrix("legacy"),
      shadow: await runMatrix("shadow"),
      enforce: await runMatrix("enforce"),
    };
  }, 120_000);

  afterAll(() => {
    (env as any).API_AUTH_MIGRATION = originalMigration;
    (env as any).ADMIN_API_KEY = originalAdminApiKey;
  });

  it("legacy matches the captured baseline except documented divergences", () => {
    const baseline = { ...matrices.legacy };
    for (const [cell, { legacy }] of Object.entries(divergences)) {
      if (legacy === undefined) continue;
      expect(baseline[cell]).toBe(legacy);
      delete baseline[cell];
    }
    expect(baseline).toMatchSnapshot();
  });

  it("shadow is byte-identical to legacy", () => {
    expect(matrices.shadow).toEqual(matrices.legacy);
  });

  it("enforce matches legacy except documented divergences", () => {
    const expected = { ...matrices.legacy };
    for (const [cell, { enforce }] of Object.entries(divergences)) {
      if (enforce !== undefined) expected[cell] = enforce;
    }
    expect(matrices.enforce).toEqual(expected);
  });

  it("covers every public route", () => {
    const denied = (route: string) =>
      denylistPrefixes.some(
        (prefix) => route === prefix || route.startsWith(`${prefix}/`),
      );
    const walked = new Set(walkRoutes().filter((route) => !denied(route)));
    const tabled = new Set(
      [...projectRoutes, ...orgRoutes].map((r) => r.route),
    );
    expect(walked).toEqual(tabled);
  });
});