🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { projectRoleAccessRights } from "@langfuse/shared";
import {
allOrganizationActions,
allProjectActions,
organizationRoleAccessRights,
systemRoleAccessRights,
type ProjectAction,
} from "@langfuse/shared/rbac";
import { describe, expect, it } from "vitest";
// The project-administration actions a legacy organization API key reserves and a
// legacy project API key does not hold.
const orgReservedProjectActions: ProjectAction[] = [
"apiKeys:read",
"apiKeys:CUD",
"projectMembers:read",
"projectMembers:CUD",
"projectMembers:manageOwnership",
"project:update",
"project:delete",
];
describe("systemRoleAccessRights", () => {
it.each([
["OWNER", true],
["LEGACY_ORGANIZATION_API_KEY", true],
["ADMIN", false],
["VIEWER", false],
] as const)(
"grants protected ownership management to %s: %s",
(role, expected) => {
const actions = systemRoleAccessRights[role].policies
.filter((policy) => policy.resourceKind === "organization")
.flatMap((policy) => policy.actions);
expect(actions.includes("organizationMembers:manageOwnership")).toBe(
expected,
);
},
);
// Decision-equivalence guard: these api-key policies pin the exact action
// sets a later ticket depends on staying equal.
it("grants a legacy project API key every project action except the org-reserved admin actions", () => {
expect(systemRoleAccessRights.LEGACY_PROJECT_API_KEY.policies).toEqual([
{
resourceKind: "project",
effect: "ALLOW",
actions: allProjectActions.filter(
(action) => !orgReservedProjectActions.includes(action),
),
},
]);
});
it("grants a legacy organization API key the full org vocab plus project administration", () => {
expect(systemRoleAccessRights.LEGACY_ORGANIZATION_API_KEY.policies).toEqual(
[
{
resourceKind: "organization",
effect: "ALLOW",
actions: allOrganizationActions,
},
{
resourceKind: "project",
effect: "ALLOW",
actions: [
"project:read",
"apiKeys:read",
"apiKeys:CUD",
"projectMembers:read",
"projectMembers:CUD",
"projectMembers:manageOwnership",
"project:update",
"project:delete",
],
},
],
);
});
it("grants SCORES_INGEST exactly scores:save at project scope", () => {
expect(systemRoleAccessRights.SCORES_INGEST.policies).toEqual([
{
resourceKind: "project",
effect: "ALLOW",
actions: ["scores:save"],
},
]);
});
it("grants INGEST exactly the ingestion project actions", () => {
expect(systemRoleAccessRights.INGEST.policies).toEqual([
{
resourceKind: "project",
effect: "ALLOW",
actions: [
"traces:create",
"scores:save",
"media:create",
"project:read",
],
},
]);
});
it("grants AI_GATEWAY exactly gateway:read and gateway:invoke at org scope", () => {
expect(systemRoleAccessRights.AI_GATEWAY.policies).toEqual([
{
resourceKind: "organization",
effect: "ALLOW",
actions: ["gateway:read", "gateway:invoke"],
},
]);
});
it("grants the AI_GATEWAY membership role the AI_GATEWAY system role's access", () => {
expect(organizationRoleAccessRights.AI_GATEWAY).toEqual([
"gateway:read",
"gateway:invoke",
]);
expect(projectRoleAccessRights.AI_GATEWAY).toEqual([]);
});
it.each(["OWNER", "ADMIN", "MEMBER", "VIEWER", "NONE"] as const)(
"grants %s organizationSettings:read",
(role) => {
expect(organizationRoleAccessRights[role]).toContain(
"organizationSettings:read",
);
},
);
it.each(["OWNER", "ADMIN", "MEMBER", "VIEWER", "NONE"] as const)(
"derives %s policies from the org and project access-right tables",
(role) => {
expect(systemRoleAccessRights[role].policies).toEqual([
{
resourceKind: "organization",
effect: "ALLOW",
actions: organizationRoleAccessRights[role],
},
{
resourceKind: "project",
effect: "ALLOW",
actions: projectRoleAccessRights[role],
},
]);
},
);
it("grants VIEWER public-API reads without write actions", () => {
const viewerProjectActions = systemRoleAccessRights.VIEWER.policies
.filter((policy) => policy.resourceKind === "project")
.flatMap((policy) => policy.actions);
expect(viewerProjectActions).toContain("traces:read");
expect(viewerProjectActions).not.toContain("scores:save");
});
it("keys every definition by its own id", () => {
for (const [key, definition] of Object.entries(systemRoleAccessRights)) {
expect(definition.id).toBe(key);
}
});
it("tags api-key roles and user roles disjointly", () => {
const tagsFor = (role: keyof typeof systemRoleAccessRights) =>
systemRoleAccessRights[role].tags;
for (const role of ["SCORES_INGEST", "INGEST"] as const) {
expect(tagsFor(role)).toEqual(["principal:apiKey"]);
}
for (const role of [
"LEGACY_PROJECT_API_KEY",
"LEGACY_ORGANIZATION_API_KEY",
] as const) {
expect(tagsFor(role)).toEqual(["principal:apiKey", "legacy"]);
}
for (const role of [
"OWNER",
"ADMIN",
"MEMBER",
"VIEWER",
"NONE",
"AI_GATEWAY",
] as const) {
expect(tagsFor(role)).toContain("principal:user");
}
});
});