Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

28.9 KB
Raw Blame
import { testFeatureFlags } from "@/src/__tests__/fixtures/feature-flags";
import type { Session } from "next-auth";

// Session fixture sub-object types; casts keep the runtime fixtures unchanged
// while satisfying newer required fields on the session user type.
type SessionUser = NonNullable<Session["user"]>;
type SessionOrg = SessionUser["organizations"][number];
import { prisma } from "@langfuse/shared/src/db";
import { appRouter } from "@/src/server/api/root";
import { createInnerTRPCContext } from "@/src/server/api/trpc";
import { TRPCError } from "@trpc/server";
import { createApiKey } from "@langfuse/shared/src/server";
import {
  ApiKeyId,
  OrganizationId,
  ProjectId,
  SystemRoleId,
  UserId,
} from "@langfuse/shared/rbac";
import { getRolesForPrincipal } from "@/src/features/rbac/getRolesForPrincipal";
import { env } from "@/src/env.mjs";
import { randomUUID } from "crypto";
import { GatewayResolveCache } from "@/src/features/ai-gateway/server/resolve/gatewayResolveCache";
import { gatewayApiFormats } from "@/src/features/ai-gateway/server/provider/registry";

vi.mock("@/src/env.mjs", async (importOriginal) => {
  const actual = await importOriginal<{ env: typeof env }>();
  return { ...actual, env: { ...actual.env } };
});

const originalRoleConfig = {
  LANGFUSE_AI_GATEWAY_CACHE_RESOLVE_ENABLED:
    env.LANGFUSE_AI_GATEWAY_CACHE_RESOLVE_ENABLED,
  API_AUTH_MIGRATION: env.API_AUTH_MIGRATION,
  API_KEY_ORG_ROLES_ENABLE: env.API_KEY_ORG_ROLES_ENABLE,
  API_KEY_PROJECT_ROLES_ENABLE: env.API_KEY_PROJECT_ROLES_ENABLE,
};

beforeEach(() => {
  Object.assign(env, {
    API_AUTH_MIGRATION: "enforce",
    API_KEY_ORG_ROLES_ENABLE: "false",
    API_KEY_PROJECT_ROLES_ENABLE: "false",
  });
});

afterEach(() => {
  Object.assign(env, originalRoleConfig);
});

describe("organization API keys trpc", () => {
  const organizationId = randomUUID();

  // The session user is persisted as the API key creator, so it must exist
  // in the database (CI does not run the seeder that creates user-1).
  // createMany + skipDuplicates is atomic, so concurrently running test
  // files can ensure the user without racing each other.
  beforeAll(async () => {
    await prisma.organization.create({
      data: { id: organizationId, name: "Test Organization" },
    });
    await prisma.user.createMany({
      data: [
        {
          id: "user-1",
          name: "Demo User",
          email: "[email protected]",
        },
      ],
      skipDuplicates: true,
    });
  });

  // Instance admins bypass both RBAC and entitlement checks, so this session
  // cannot cover plan gating. `entitledOwnerCaller` / `unentitledOwnerCaller`
  // below are the non-admin owners used for that.
  const ownerSession: Session = {
    expires: "1",
    user: {
      id: "user-1",
      canCreateOrganizations: true,
      name: "Demo User",
      organizations: [
        {
          id: organizationId,
          name: "Test Organization",
          role: "OWNER",
          plan: "cloud:hobby",
          cloudConfig: undefined,
          metadata: {},
          projects: [] as SessionOrg["projects"],
        } as SessionOrg,
      ],
      featureFlags: testFeatureFlags(),
      admin: true,
    },
    environment: {} as any,
  };

  const memberSession: Session = {
    expires: "1",
    user: {
      id: "user-2",
      canCreateOrganizations: true,
      name: "Member User",
      organizations: [
        {
          id: organizationId,
          name: "Test Organization",
          role: "MEMBER",
          plan: "cloud:hobby",
          cloudConfig: undefined,
          metadata: {},
          projects: [] as SessionOrg["projects"],
        } as SessionOrg,
      ],
      featureFlags: testFeatureFlags(),
      admin: false,
    },
    environment: {} as any,
  };

  const adminSession: Session = {
    expires: "1",
    user: {
      id: "user-3",
      canCreateOrganizations: true,
      name: "Admin User",
      organizations: [
        {
          id: organizationId,
          name: "Test Organization",
          role: "ADMIN",
          plan: "cloud:hobby",
          cloudConfig: undefined,
          metadata: {},
          projects: [] as SessionOrg["projects"],
        } as SessionOrg,
      ],
      featureFlags: testFeatureFlags(),
      admin: false,
    },
    environment: {} as any,
  };

  const ownerCtx = createInnerTRPCContext({
    session: ownerSession,
    headers: {},
  });
  const ownerCaller = appRouter.createCaller({ ...ownerCtx, prisma });

  const memberCtx = createInnerTRPCContext({
    session: memberSession,
    headers: {},
  });
  const memberCaller = appRouter.createCaller({ ...memberCtx, prisma });

  const adminCtx = createInnerTRPCContext({
    session: adminSession,
    headers: {},
  });
  const adminCaller = appRouter.createCaller({ ...adminCtx, prisma });

  const unAuthedCtx = createInnerTRPCContext({ session: null, headers: {} });
  const unAuthedCaller = appRouter.createCaller({ ...unAuthedCtx, prisma });

  // Organization-scoped API keys are a paid feature (`admin-api` entitlement).
  // These owners are deliberately not instance admins so the plan gate applies.
  const ownerSessionOnPlan = (plan: SessionOrg["plan"]): Session => ({
    expires: "1",
    user: {
      id: "user-1",
      canCreateOrganizations: true,
      name: "Demo User",
      organizations: [
        {
          id: organizationId,
          name: "Test Organization",
          role: "OWNER",
          plan,
          cloudConfig: undefined,
          metadata: {},
          projects: [] as SessionOrg["projects"],
        } as SessionOrg,
      ],
      featureFlags: testFeatureFlags(),
      admin: false,
    },
    environment: {} as any,
  });

  const callerForSession = (session: Session) =>
    appRouter.createCaller({
      ...createInnerTRPCContext({ session, headers: {} }),
      prisma,
    });

  // cloud:hobby lacks `admin-api`; cloud:team carries it.
  const unentitledOwnerCaller = callerForSession(
    ownerSessionOnPlan("cloud:hobby"),
  );
  const entitledOwnerCaller = callerForSession(
    ownerSessionOnPlan("cloud:team"),
  );

  describe("organizationApiKeys.byOrganizationId", () => {
    it("owner can fetch organization API keys", async () => {
      // Create a test API key first
      const apiKeyResult = await ownerCaller.organizationApiKeys.create({
        orgId: organizationId,
        name: "Test API Key",
      });

      expect(apiKeyResult).toBeDefined();
      expect(apiKeyResult.secretKey).toBeDefined();
      expect(apiKeyResult.publicKey).toBeDefined();

      // Now fetch the keys
      const apiKeys = await ownerCaller.organizationApiKeys.byOrganizationId({
        orgId: organizationId,
      });

      expect(apiKeys.length).toBeGreaterThanOrEqual(1);
      const newKey = apiKeys.find((key) => key.id === apiKeyResult.id);
      expect(newKey?.note).toBe("Test API Key");
      expect(newKey?.publicKey).toBe(apiKeyResult.publicKey);
      expect(newKey?.displaySecretKey).toBeDefined();
    });

    it("filters in-app agent API keys", async () => {
      const inAppAgentKey = await createApiKey(prisma, {
        tenant: OrganizationId(organizationId),
        scope: OrganizationId(organizationId),
        role: SystemRoleId("LEGACY_ORGANIZATION_API_KEY"),
        createdBy: UserId("user-1"),
        name: "In-app agent key hidden from org UI",
        isInAppAgentKey: true,
      });

      const apiKeys = await ownerCaller.organizationApiKeys.byOrganizationId({
        orgId: organizationId,
      });

      expect(apiKeys.map((key) => key.id)).not.toContain(inAppAgentKey.id);
      expect(apiKeys.map((key) => key.note)).not.toContain(
        "In-app agent key hidden from org UI",
      );
    });

    it("filters AI Gateway keys", async () => {
      const gatewayKey = await createApiKey(prisma, {
        tenant: OrganizationId(organizationId),
        scope: OrganizationId(organizationId),
        role: SystemRoleId("AI_GATEWAY"),
        createdBy: UserId("user-1"),
        name: "Gateway key",
      });
      const ingestKey = await createApiKey(prisma, {
        tenant: OrganizationId(organizationId),
        scope: OrganizationId(organizationId),
        role: SystemRoleId("INGEST"),
        createdBy: UserId("user-1"),
        name: "Ingestion key",
      });

      const apiKeys = await ownerCaller.organizationApiKeys.byOrganizationId({
        orgId: organizationId,
      });
      const ids = apiKeys.map((key) => key.id);

      expect(ids).not.toContain(gatewayKey.id);
      expect(ids).toContain(ingestKey.id);

      const allApiKeys = await ownerCaller.organizationApiKeys.byOrganizationId(
        { orgId: organizationId, includeGatewayKeys: true },
      );
      const allIds = allApiKeys.map((key) => key.id);

      expect(allIds).toContain(gatewayKey.id);
      expect(allIds).toContain(ingestKey.id);
    });

    it("regular member cannot fetch organization API keys", async () => {
      await expect(
        memberCaller.organizationApiKeys.byOrganizationId({
          orgId: organizationId,
        }),
      ).rejects.toThrow(TRPCError);
    });

    it("admin cannot fetch organization API keys", async () => {
      await expect(
        adminCaller.organizationApiKeys.byOrganizationId({
          orgId: organizationId,
        }),
      ).rejects.toThrow(TRPCError);
    });

    it("unauthenticated user cannot fetch organization API keys", async () => {
      await expect(
        unAuthedCaller.organizationApiKeys.byOrganizationId({
          orgId: organizationId,
        }),
      ).rejects.toThrow(TRPCError);
    });
  });

  describe("organizationApiKeys.create", () => {
    it("owner can create organization API keys", async () => {
      const apiKeyResult = await ownerCaller.organizationApiKeys.create({
        orgId: organizationId,
        name: "Test API Key",
      });

      expect(apiKeyResult).toBeDefined();
      expect(apiKeyResult.secretKey).toBeDefined();
      expect(apiKeyResult.publicKey).toBeDefined();
      expect(apiKeyResult.note).toBe("Test API Key");
    });

    it("stores the creating user and returns it in the list", async () => {
      const apiKeyResult = await ownerCaller.organizationApiKeys.create({
        orgId: organizationId,
        name: "Key for creator attribution test",
      });

      const dbKey = await prisma.apiKey.findUniqueOrThrow({
        where: { id: apiKeyResult.id },
      });
      expect(dbKey.createdByUserId).toBe("user-1");
      expect(dbKey.createdByApiKeyId).toBeNull();

      const apiKeys = await ownerCaller.organizationApiKeys.byOrganizationId({
        orgId: organizationId,
      });
      const listedKey = apiKeys.find((key) => key.id === apiKeyResult.id);
      expect(listedKey?.createdByUser?.id).toBe("user-1");
      expect(listedKey?.createdByApiKey).toBeNull();
    });

    it("rejects an expiration date in the past", async () => {
      await expect(
        ownerCaller.organizationApiKeys.create({
          orgId: organizationId,
          expiresAt: new Date(Date.now() - 60_000),
        }),
      ).rejects.toMatchObject({ code: "BAD_REQUEST" });
    });

    it("regular member cannot create organization API keys", async () => {
      await expect(
        memberCaller.organizationApiKeys.create({
          orgId: organizationId,
          name: "Test API Key",
        }),
      ).rejects.toThrow(TRPCError);
    });

    it("admin cannot create organization API keys", async () => {
      await expect(
        adminCaller.organizationApiKeys.create({
          orgId: organizationId,
          name: "Test API Key",
        }),
      ).rejects.toThrow(TRPCError);
    });

    it("unauthenticated user cannot create organization API keys", async () => {
      await expect(
        unAuthedCaller.organizationApiKeys.create({
          orgId: organizationId,
          name: "Test API Key",
        }),
      ).rejects.toThrow(TRPCError);
    });

    it("owner on a plan without admin-api cannot create organization API keys", async () => {
      // Unique note so the persistence assertion stays independent of other
      // tests and of keys left behind by earlier runs.
      const name = `unentitled-create-${randomUUID()}`;

      await expect(
        unentitledOwnerCaller.organizationApiKeys.create({
          orgId: organizationId,
          name,
        }),
      ).rejects.toThrow(
        expect.objectContaining({
          code: "FORBIDDEN",
          message: expect.stringContaining("admin-api"),
        }),
      );

      // The rejection must happen before the key is persisted, otherwise the
      // plan gate would only hide a key that already works.
      await expect(
        prisma.apiKey.findFirst({
          where: { orgId: organizationId, note: name },
        }),
      ).resolves.toBeNull();
    });

    it("owner on a plan with admin-api can create organization API keys", async () => {
      const apiKeyResult = await entitledOwnerCaller.organizationApiKeys.create(
        {
          orgId: organizationId,
          name: "Entitled plan key",
        },
      );

      expect(apiKeyResult.secretKey).toBeDefined();
      expect(apiKeyResult.publicKey).toBeDefined();
    });
  });

  describe("organizationApiKeys.updateName", () => {
    it("owner can update API key note", async () => {
      // Create a key first
      const apiKeyResult = await ownerCaller.organizationApiKeys.create({
        orgId: organizationId,
        name: "Original Note",
      });

      // Update the note
      await ownerCaller.organizationApiKeys.updateName({
        orgId: organizationId,
        keyId: apiKeyResult.id,
        name: "Updated Note",
      });

      // Fetch to verify
      const apiKeys = await ownerCaller.organizationApiKeys.byOrganizationId({
        orgId: organizationId,
      });

      expect(apiKeys.length).toBeGreaterThanOrEqual(1);
      const updatedKey = apiKeys.find((key) => key.id === apiKeyResult.id);
      expect(updatedKey?.note).toBe("Updated Note");
    });

    it("evicts a renamed gateway key from the gateway cache", async () => {
      Object.assign(env, { LANGFUSE_AI_GATEWAY_CACHE_RESOLVE_ENABLED: "true" });
      const gatewayKey = await createApiKey(prisma, {
        tenant: OrganizationId(organizationId),
        scope: OrganizationId(organizationId),
        role: SystemRoleId("AI_GATEWAY"),
        createdBy: UserId("user-1"),
        name: "Original gateway key",
      });
      const { fastHashedSecretKey } = await prisma.apiKey.findUniqueOrThrow({
        where: { id: gatewayKey.id },
        select: { fastHashedSecretKey: true },
      });
      const cache = new GatewayResolveCache();
      const cacheEntry = {
        fastHashedSecretKey: fastHashedSecretKey!,
        apiFormat: gatewayApiFormats[0],
      };
      await cache.set({
        ...cacheEntry,
        context: {
          organizationId,
          apiKeyId: gatewayKey.id,
          userId: null,
          apiKeyName: "Original gateway key",
          userName: null,
          userEmail: null,
          keyMetadata: {},
          ingestionProjectId: null,
          ingestionMode: null,
          plan: "cloud:hobby",
          rateLimitOverrides: [],
          connections: [],
        },
      });
      expect(await cache.get(cacheEntry)).not.toBeNull();

      await ownerCaller.organizationApiKeys.updateName({
        orgId: organizationId,
        keyId: gatewayKey.id,
        name: "Renamed gateway key",
      });

      expect(await cache.get(cacheEntry)).toBeNull();
    });

    it("rejects a blank name for a gateway key but not for other keys", async () => {
      const gatewayKey = await createApiKey(prisma, {
        tenant: OrganizationId(organizationId),
        scope: OrganizationId(organizationId),
        role: SystemRoleId("AI_GATEWAY"),
        createdBy: UserId("user-1"),
        name: "Named gateway key",
      });

      await expect(
        ownerCaller.organizationApiKeys.updateName({
          orgId: organizationId,
          keyId: gatewayKey.id,
          name: "  ",
        }),
      ).rejects.toMatchObject({ code: "BAD_REQUEST" });
      await expect(
        prisma.apiKey.findUniqueOrThrow({ where: { id: gatewayKey.id } }),
      ).resolves.toMatchObject({ note: "Named gateway key" });

      const otherKey = await ownerCaller.organizationApiKeys.create({
        orgId: organizationId,
        name: "Named key",
      });
      await ownerCaller.organizationApiKeys.updateName({
        orgId: organizationId,
        keyId: otherKey.id,
        name: "",
      });
    });

    it("returns NOT_FOUND for a missing API key", async () => {
      await expect(
        ownerCaller.organizationApiKeys.updateName({
          orgId: organizationId,
          keyId: randomUUID(),
          name: "Updated Note",
        }),
      ).rejects.toMatchObject({ code: "NOT_FOUND" });
    });

    it("does not update in-app agent API keys", async () => {
      const inAppAgentKey = await createApiKey(prisma, {
        tenant: OrganizationId(organizationId),
        scope: OrganizationId(organizationId),
        role: SystemRoleId("LEGACY_ORGANIZATION_API_KEY"),
        createdBy: UserId("user-1"),
        name: "Original in-app agent note",
        isInAppAgentKey: true,
      });

      await expect(
        ownerCaller.organizationApiKeys.updateName({
          orgId: organizationId,
          keyId: inAppAgentKey.id,
          name: "Updated in-app agent note",
        }),
      ).rejects.toThrow();

      const persistedKey = await prisma.apiKey.findUniqueOrThrow({
        where: { id: inAppAgentKey.id },
      });
      expect(persistedKey.note).toBe("Original in-app agent note");
    });

    it("regular member cannot update API key note", async () => {
      // Create a key as owner
      const apiKeyResult = await ownerCaller.organizationApiKeys.create({
        orgId: organizationId,
        name: "Original Note",
      });

      // Try to update as member
      await expect(
        memberCaller.organizationApiKeys.updateName({
          orgId: organizationId,
          keyId: apiKeyResult.id,
          name: "Updated Note",
        }),
      ).rejects.toThrow(TRPCError);
    });

    it("admin cannot update API key note", async () => {
      // Create a key as owner
      const apiKeyResult = await ownerCaller.organizationApiKeys.create({
        orgId: organizationId,
        name: "Original Note",
      });

      // Try to update as admin
      await expect(
        adminCaller.organizationApiKeys.updateName({
          orgId: organizationId,
          keyId: apiKeyResult.id,
          name: "Updated Note",
        }),
      ).rejects.toThrow(TRPCError);
    });
  });

  describe("organizationApiKeys.delete", () => {
    it("owner can delete API key", async () => {
      // Create a key first
      const apiKeyResult = await ownerCaller.organizationApiKeys.create({
        orgId: organizationId,
        name: "To Be Deleted",
      });

      // Delete the key
      await ownerCaller.organizationApiKeys.delete({
        orgId: organizationId,
        id: apiKeyResult.id,
      });

      // Verify it's gone
      const apiKeys = await ownerCaller.organizationApiKeys.byOrganizationId({
        orgId: organizationId,
      });

      const deletedKey = apiKeys.find((key) => key.id === apiKeyResult.id);
      expect(deletedKey).toBeUndefined();
    });

    it("stops a deleted gateway key from resolving through the gateway cache", async () => {
      Object.assign(env, { LANGFUSE_AI_GATEWAY_CACHE_RESOLVE_ENABLED: "true" });
      const gatewayKey = await createApiKey(prisma, {
        tenant: OrganizationId(organizationId),
        scope: OrganizationId(organizationId),
        role: SystemRoleId("AI_GATEWAY"),
        createdBy: UserId("user-1"),
        name: "Cached gateway key",
      });
      const { fastHashedSecretKey } = await prisma.apiKey.findUniqueOrThrow({
        where: { id: gatewayKey.id },
        select: { fastHashedSecretKey: true },
      });
      const cache = new GatewayResolveCache();
      const cacheEntry = {
        fastHashedSecretKey: fastHashedSecretKey!,
        apiFormat: gatewayApiFormats[0],
      };
      await cache.set({
        ...cacheEntry,
        context: {
          organizationId,
          apiKeyId: gatewayKey.id,
          userId: null,
          apiKeyName: "Cached gateway key",
          userName: null,
          userEmail: null,
          keyMetadata: {},
          ingestionProjectId: null,
          ingestionMode: null,
          plan: "cloud:hobby",
          rateLimitOverrides: [],
          connections: [],
        },
      });
      expect(await cache.get(cacheEntry)).not.toBeNull();

      await ownerCaller.organizationApiKeys.delete({
        orgId: organizationId,
        id: gatewayKey.id,
      });

      expect(await cache.get(cacheEntry)).toBeNull();
    });

    it("returns NOT_FOUND for a missing API key", async () => {
      await expect(
        ownerCaller.organizationApiKeys.delete({
          orgId: organizationId,
          id: randomUUID(),
        }),
      ).rejects.toMatchObject({ code: "NOT_FOUND" });
    });

    it("does not delete in-app agent API keys", async () => {
      const inAppAgentKey = await createApiKey(prisma, {
        tenant: OrganizationId(organizationId),
        scope: OrganizationId(organizationId),
        role: SystemRoleId("LEGACY_ORGANIZATION_API_KEY"),
        createdBy: UserId("user-1"),
        isInAppAgentKey: true,
      });

      await expect(
        ownerCaller.organizationApiKeys.delete({
          orgId: organizationId,
          id: inAppAgentKey.id,
        }),
      ).resolves.toBe(false);

      await expect(
        prisma.apiKey.findUniqueOrThrow({ where: { id: inAppAgentKey.id } }),
      ).resolves.toBeDefined();
    });

    it("regular member cannot delete API key", async () => {
      // Create a key as owner
      const apiKeyResult = await ownerCaller.organizationApiKeys.create({
        orgId: organizationId,
        name: "To Be Deleted",
      });

      // Try to delete as member
      await expect(
        memberCaller.organizationApiKeys.delete({
          orgId: organizationId,
          id: apiKeyResult.id,
        }),
      ).rejects.toThrow(TRPCError);
    });

    it("admin cannot delete API key", async () => {
      // Create a key as owner
      const apiKeyResult = await ownerCaller.organizationApiKeys.create({
        orgId: organizationId,
        name: "To Be Deleted",
      });

      // Try to delete as admin
      await expect(
        adminCaller.organizationApiKeys.delete({
          orgId: organizationId,
          id: apiKeyResult.id,
        }),
      ).rejects.toThrow(TRPCError);
    });

    it("owner on a plan without admin-api can still list and revoke existing keys", async () => {
      // Only creation is plan-gated. A downgraded organization must keep the
      // ability to see and revoke keys that were issued while entitled,
      // otherwise a downgrade would strand live credentials.
      const existingKey = await createApiKey(prisma, {
        tenant: OrganizationId(organizationId),
        scope: OrganizationId(organizationId),
        role: SystemRoleId("LEGACY_ORGANIZATION_API_KEY"),
        createdBy: UserId("user-1"),
        name: "Issued before downgrade",
      });

      const apiKeys =
        await unentitledOwnerCaller.organizationApiKeys.byOrganizationId({
          orgId: organizationId,
        });
      expect(apiKeys.map((key) => key.id)).toContain(existingKey.id);

      await unentitledOwnerCaller.organizationApiKeys.delete({
        orgId: organizationId,
        id: existingKey.id,
      });

      await expect(
        prisma.apiKey.findUnique({ where: { id: existingKey.id } }),
      ).resolves.toBeNull();
    });
  });

  describe("organizationApiKeys.create role scoping", () => {
    const ownerCallerForOrg = (orgId: string) =>
      callerForSession({
        expires: "1",
        user: {
          id: "user-1",
          canCreateOrganizations: true,
          name: "Demo User",
          organizations: [
            {
              id: orgId,
              name: "Scoped Org",
              role: "OWNER",
              plan: "cloud:team",
              cloudConfig: undefined,
              metadata: {},
              projects: [] as SessionOrg["projects"],
            } as SessionOrg,
          ],
          featureFlags: testFeatureFlags(),
          admin: false,
        },
        environment: {} as any,
      });

    // An org key owns a single assignment on the organization, and the resolver
    // expands that owner to the project-kind wildcard, so a project-capable role
    // reaches every project of the org rather than a chosen subset.
    it("creates an org key that reaches every project via the project wildcard", async () => {
      Object.assign(env, { API_KEY_ORG_ROLES_ENABLE: "true" });
      const orgId = `org-${randomUUID()}`;
      await prisma.organization.create({ data: { id: orgId, name: "Scoped" } });

      const key = await ownerCallerForOrg(orgId).organizationApiKeys.create({
        orgId,
        role: "ADMIN",
      });

      const projectResources = (
        await getRolesForPrincipal(prisma, ApiKeyId(key.id))
      )
        .flatMap((role) => role.policies)
        .filter((policy) => policy.id.endsWith(":project"))
        .flatMap((policy) => policy.resources);

      expect(projectResources).toContain(ProjectId("*"));
    });

    it.each([
      "ADMIN",
      "VIEWER",
      "INGEST",
      "SCORES_INGEST",
      "AI_GATEWAY",
    ] as const)(
      "creates an organization key with %s when role selection is enabled",
      async (role) => {
        Object.assign(env, { API_KEY_ORG_ROLES_ENABLE: "true" });
        const orgId = `org-${randomUUID()}`;
        await prisma.organization.create({
          data: { id: orgId, name: "Scoped" },
        });
        const key = await ownerCallerForOrg(orgId).organizationApiKeys.create({
          orgId,
          name: `${role} key`,
          role,
        });
        expect(
          await prisma.roleAssignment.findFirstOrThrow({
            where: { apiKeyId: key.id },
          }),
        ).toMatchObject({
          systemRole: role,
          orgId,
          ownerId: `organization/${orgId}`,
          projectId: null,
        });
      },
    );

    it("rejects an AI_GATEWAY key without a name", async () => {
      Object.assign(env, { API_KEY_ORG_ROLES_ENABLE: "true" });
      const orgId = `org-${randomUUID()}`;
      await prisma.organization.create({
        data: { id: orgId, name: "Scoped" },
      });

      await expect(
        ownerCallerForOrg(orgId).organizationApiKeys.create({
          orgId,
          name: "  ",
          role: "AI_GATEWAY",
        }),
      ).rejects.toMatchObject({ code: "BAD_REQUEST" });
      await expect(prisma.apiKey.count({ where: { orgId } })).resolves.toBe(0);
    });

    describe.each([
      { migration: "legacy", enabled: "false" },
      { migration: "legacy", enabled: "true" },
      { migration: "shadow", enabled: "false" },
      { migration: "shadow", enabled: "true" },
      { migration: "enforce", enabled: "false" },
      { migration: "enforce", enabled: "true" },
    ])(
      "role creation with $migration and exposure $enabled",
      ({ migration, enabled }) => {
        beforeEach(() => {
          Object.assign(env, {
            API_AUTH_MIGRATION: migration,
            API_KEY_ORG_ROLES_ENABLE: enabled,
            API_KEY_PROJECT_ROLES_ENABLE: "true",
          });
        });

        it.each([{}, { role: undefined }, { role: null }])(
          "defaults %j to the legacy role",
          async (input) => {
            const orgId = `org-${randomUUID()}`;
            await prisma.organization.create({
              data: { id: orgId, name: "Scoped" },
            });
            const key = await ownerCallerForOrg(
              orgId,
            ).organizationApiKeys.create({ orgId, ...input });
            expect(
              await prisma.roleAssignment.findFirstOrThrow({
                where: { apiKeyId: key.id },
              }),
            ).toMatchObject({
              systemRole: "LEGACY_ORGANIZATION_API_KEY",
              orgId,
              ownerId: `organization/${orgId}`,
              projectId: null,
            });
          },
        );

        it("rejects explicit legacy, empty, invalid and disabled roles without creating keys", async () => {
          const orgId = `org-${randomUUID()}`;
          await prisma.organization.create({
            data: { id: orgId, name: "Scoped" },
          });
          const caller = ownerCallerForOrg(orgId);
          const rejectedRoles = [
            "LEGACY_PROJECT_API_KEY",
            "LEGACY_ORGANIZATION_API_KEY",
            "",
            "OWNER",
            "invalid",
          ];
          if (migration !== "enforce" || enabled === "false")
            rejectedRoles.push(
              "ADMIN",
              "VIEWER",
              "INGEST",
              "SCORES_INGEST",
              "AI_GATEWAY",
            );
          for (const role of rejectedRoles) {
            await expect(
              caller.organizationApiKeys.create({ orgId, role: role as never }),
            ).rejects.toMatchObject({ code: "BAD_REQUEST" });
          }
          expect(await prisma.apiKey.count({ where: { orgId } })).toBe(0);
        });
      },
    );
  });
});