Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

refactor(evals): rename evaluation RBAC scopes (#16648)

Tobias Wochinger1124345b79bb
diff --git a/packages/shared/src/features/rbac/projectAccessRights.ts b/packages/shared/src/features/rbac/projectAccessRights.ts
index 4fe8643b35..21c6883cbb 100644
--- a/packages/shared/src/features/rbac/projectAccessRights.ts
+++ b/packages/shared/src/features/rbac/projectAccessRights.ts
@@ -44,10 +44,10 @@ export const projectScopes = [
"batchExports:create",
"batchExports:read",
- "evalTemplate:CUD",
- "evalTemplate:read",
- "evalJob:read",
- "evalJob:CUD",
+ "evaluator:CUD",
+ "evaluator:read",
+ "evaluationRule:read",
+ "evaluationRule:CUD",
"evalJobExecution:read",
"evalDefaultModel:read",
"evalDefaultModel:CUD",
@@ -109,10 +109,10 @@ export const projectRoleAccessRights: Record<Role, ProjectScope[]> = {
"prompts:read",
"promptProtectedLabels:CUD",
"models:CUD",
- "evalTemplate:CUD",
- "evalTemplate:read",
- "evalJob:CUD",
- "evalJob:read",
+ "evaluator:CUD",
+ "evaluator:read",
+ "evaluationRule:CUD",
+ "evaluationRule:read",
"evalJobExecution:read",
"evalDefaultModel:CUD",
"evalDefaultModel:read",
@@ -166,10 +166,10 @@ export const projectRoleAccessRights: Record<Role, ProjectScope[]> = {
"prompts:read",
"promptProtectedLabels:CUD",
"models:CUD",
- "evalTemplate:CUD",
- "evalTemplate:read",
- "evalJob:CUD",
- "evalJob:read",
+ "evaluator:CUD",
+ "evaluator:read",
+ "evaluationRule:CUD",
+ "evaluationRule:read",
"evalJobExecution:read",
"evalDefaultModel:CUD",
"evalDefaultModel:read",
@@ -216,10 +216,10 @@ export const projectRoleAccessRights: Record<Role, ProjectScope[]> = {
"datasets:CUD",
"prompts:CUD",
"prompts:read",
- "evalTemplate:CUD",
- "evalTemplate:read",
- "evalJob:read",
- "evalJob:CUD",
+ "evaluator:CUD",
+ "evaluator:read",
+ "evaluationRule:read",
+ "evaluationRule:CUD",
"evalJobExecution:read",
"evalDefaultModel:read",
"evalDefaultModel:CUD",
@@ -249,9 +249,9 @@ export const projectRoleAccessRights: Record<Role, ProjectScope[]> = {
VIEWER: [
"project:read",
"prompts:read",
- "evalTemplate:read",
+ "evaluator:read",
"scoreConfigs:read",
- "evalJob:read",
+ "evaluationRule:read",
"evalJobExecution:read",
"evalDefaultModel:read",
"datasets:read",
diff --git a/packages/shared/src/in-app-agent/server/mcpPolicy.ts b/packages/shared/src/in-app-agent/server/mcpPolicy.ts
index e82c123699..9df9515268 100644
--- a/packages/shared/src/in-app-agent/server/mcpPolicy.ts
+++ b/packages/shared/src/in-app-agent/server/mcpPolicy.ts
@@ -134,55 +134,55 @@ export const IN_APP_AGENT_LANGFUSE_MCP_TOOL_POLICIES = {
},
listEvaluators: {
approval: "auto",
- availability: { scope: "evalTemplate:read" },
+ availability: { scope: "evaluator:read" },
},
listManagedEvaluatorTemplates: {
approval: "auto",
- availability: { scope: "evalTemplate:read" },
+ availability: { scope: "evaluator:read" },
},
getEvaluator: {
approval: "auto",
- availability: { scope: "evalTemplate:read" },
+ availability: { scope: "evaluator:read" },
},
createEvaluator: {
approval: "approval",
- availability: { scope: "evalTemplate:CUD" },
+ availability: { scope: "evaluator:CUD" },
},
updateEvaluator: {
approval: "approval",
- availability: { scope: "evalTemplate:CUD" },
+ availability: { scope: "evaluator:CUD" },
},
deleteEvaluator: {
approval: "approval",
- availability: { scope: "evalTemplate:CUD" },
+ availability: { scope: "evaluator:CUD" },
},
listEvaluationRules: {
approval: "auto",
- availability: { scope: "evalJob:read" },
+ availability: { scope: "evaluationRule:read" },
},
getEvaluationRule: {
approval: "auto",
- availability: { scope: "evalJob:read" },
+ availability: { scope: "evaluationRule:read" },
},
createEvaluationRule: {
approval: "approval",
- availability: { scope: "evalJob:CUD" },
+ availability: { scope: "evaluationRule:CUD" },
},
updateEvaluationRule: {
approval: "approval",
- availability: { scope: "evalJob:CUD" },
+ availability: { scope: "evaluationRule:CUD" },
},
attachEvaluatorToEvaluationRule: {
approval: "approval",
- availability: { scope: "evalJob:CUD" },
+ availability: { scope: "evaluationRule:CUD" },
},
detachEvaluatorFromEvaluationRule: {
approval: "approval",
- availability: { scope: "evalJob:CUD" },
+ availability: { scope: "evaluationRule:CUD" },
},
deleteEvaluationRule: {
approval: "approval",
- availability: { scope: "evalJob:CUD" },
+ availability: { scope: "evaluationRule:CUD" },
},
listExperiments: {
approval: "auto",
diff --git a/web/src/components/deleteButton.tsx b/web/src/components/deleteButton.tsx
index bf1491e981..9a66b534ae 100644
--- a/web/src/components/deleteButton.tsx
+++ b/web/src/components/deleteButton.tsx
@@ -411,7 +411,7 @@ export function DeleteEvalConfigButton(props: DeleteButtonProps) {
const {
itemId,
projectId,
- scope = "evalJob:CUD",
+ scope = "evaluationRule:CUD",
invalidateFunc = () => utils.evals.invalidate(),
} = props;
diff --git a/web/src/components/layouts/routes.tsx b/web/src/components/layouts/routes.tsx
index 1ff1a9ff45..c861c744ec 100644
--- a/web/src/components/layouts/routes.tsx
+++ b/web/src/components/layouts/routes.tsx
@@ -169,7 +169,7 @@ export const ROUTES: Route[] = [
title: "Evaluators",
icon: Lightbulb,
productModule: "evaluation",
- projectRbacScopes: ["evalJob:read"],
+ projectRbacScopes: ["evaluator:read", "evaluationRule:read"],
group: RouteGroup.Evaluation,
section: RouteSection.Main,
pathname: `/project/[projectId]/evals`,
diff --git a/web/src/components/table/peek/peek-evaluator-config-detail.tsx b/web/src/components/table/peek/peek-evaluator-config-detail.tsx
index 7157ed0866..1c29751fd5 100644
--- a/web/src/components/table/peek/peek-evaluator-config-detail.tsx
+++ b/web/src/components/table/peek/peek-evaluator-config-detail.tsx
@@ -51,7 +51,10 @@ const PeekViewEvaluatorConfigDetail = ({
evaluatorId: evalConfig?.id,
evaluator: evalConfig,
});
- const hasAccess = useHasProjectAccess({ projectId, scope: "evalJob:CUD" });
+ const hasAccess = useHasProjectAccess({
+ projectId,
+ scope: "evaluationRule:CUD",
+ });
if (!evalConfig) {
return <Skeleton className="h-full w-full rounded-none" />;
diff --git a/web/src/features/batch-actions/server/runEvaluationRouter.ts b/web/src/features/batch-actions/server/runEvaluationRouter.ts
index e82fc12b94..e7927e5730 100644
--- a/web/src/features/batch-actions/server/runEvaluationRouter.ts
+++ b/web/src/features/batch-actions/server/runEvaluationRouter.ts
@@ -32,7 +32,7 @@ export const runEvaluationRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
- scope: "evalJob:CUD",
+ scope: "evaluationRule:CUD",
});
const {
diff --git a/web/src/features/evals/components/deactivate-config.tsx b/web/src/features/evals/components/deactivate-config.tsx
index 85be7a5f0b..b0eb34a1b8 100644
--- a/web/src/features/evals/components/deactivate-config.tsx
+++ b/web/src/features/evals/components/deactivate-config.tsx
@@ -25,7 +25,10 @@ export function DeactivateEvalConfig({
onStatusChange?: () => void;
}) {
const utils = api.useUtils();
- const hasAccess = useHasProjectAccess({ projectId, scope: "evalJob:CUD" });
+ const hasAccess = useHasProjectAccess({
+ projectId,
+ scope: "evaluationRule:CUD",
+ });
const { allowLegacy } = useEvalCapabilities(projectId);
const [isOpen, setIsOpen] = useState(false);
const capture = usePostHogClientCapture();
diff --git a/web/src/features/evals/components/eval-template-detail.tsx b/web/src/features/evals/components/eval-template-detail.tsx
index 6621c5d511..ac0f852dbf 100644
--- a/web/src/features/evals/components/eval-template-detail.tsx
+++ b/web/src/features/evals/components/eval-template-detail.tsx
@@ -31,7 +31,7 @@ export const EvalTemplateDetail = () => {
const capture = usePostHogClientCapture();
const hasDeleteAccess = useHasProjectAccess({
projectId,
- scope: "evalTemplate:CUD",
+ scope: "evaluator:CUD",
});
// get the current template by id
@@ -231,7 +231,7 @@ function UpdateTemplate({
}) {
const hasAccess = useHasProjectAccess({
projectId,
- scope: "evalTemplate:CUD",
+ scope: "evaluator:CUD",
});
const capture = usePostHogClientCapture();
diff --git a/web/src/features/evals/components/eval-templates-table.tsx b/web/src/features/evals/components/eval-templates-table.tsx
index 0a3585f516..4b1b44dfb1 100644
--- a/web/src/features/evals/components/eval-templates-table.tsx
+++ b/web/src/features/evals/components/eval-templates-table.tsx
@@ -148,7 +148,7 @@ const EvalTemplateRowActionsMenu = ({
const utils = api.useUtils();
const hasTemplateWriteAccess = useHasProjectAccess({
projectId,
- scope: "evalTemplate:CUD",
+ scope: "evaluator:CUD",
});
return (
@@ -255,7 +255,10 @@ export default function EvalsTemplateTable({
searchQuery: searchQuery,
});
- const hasAccess = useHasProjectAccess({ projectId, scope: "evalJob:CUD" });
+ const hasAccess = useHasProjectAccess({
+ projectId,
+ scope: "evaluator:CUD",
+ });
const totalCount = templates.data?.totalCount ?? null;
diff --git a/web/src/features/evals/components/evaluator-table.tsx b/web/src/features/evals/components/evaluator-table.tsx
index 8418107313..3c17e38cc8 100644
--- a/web/src/features/evals/components/evaluator-table.tsx
+++ b/web/src/features/evals/components/evaluator-table.tsx
@@ -126,7 +126,10 @@ export default function EvaluatorTable({ projectId }: { projectId: string }) {
},
);
- const hasAccess = useHasProjectAccess({ projectId, scope: "evalJob:CUD" });
+ const hasAccess = useHasProjectAccess({
+ projectId,
+ scope: "evaluationRule:CUD",
+ });
// Deprecated evaluators are read-only where new legacy setups are not
// allowed (cloud); self-hosted deployments keep editing them.
const { allowLegacy } = useEvalCapabilities(projectId);
diff --git a/web/src/features/evals/pages/evaluators.tsx b/web/src/features/evals/pages/evaluators.tsx
index 6d909a4b28..c0efd92680 100644
--- a/web/src/features/evals/pages/evaluators.tsx
+++ b/web/src/features/evals/pages/evaluators.tsx
@@ -24,12 +24,12 @@ export default function EvaluatorsPage() {
);
const hasWriteAccess = useHasProjectAccess({
projectId,
- scope: "evalJob:CUD",
+ scope: "evaluationRule:CUD",
});
const hasReadAccess = useHasProjectAccess({
projectId,
- scope: "evalJob:read",
+ scope: "evaluationRule:read",
});
// Fetch counts of evaluator configs and templates
diff --git a/web/src/features/evals/pages/new-evaluator.tsx b/web/src/features/evals/pages/new-evaluator.tsx
index bd1800b1e6..cf017fff91 100644
--- a/web/src/features/evals/pages/new-evaluator.tsx
+++ b/web/src/features/evals/pages/new-evaluator.tsx
@@ -51,10 +51,15 @@ export default function NewEvaluatorPage() {
const hasDefaultModel =
!!defaultModelQuery.data || defaultModelConfiguredInFlow;
- const hasAccess = useHasProjectAccess({
+ const hasEvaluatorReadAccess = useHasProjectAccess({
projectId,
- scope: "evalTemplate:CUD",
+ scope: "evaluator:read",
});
+ const hasEvaluationRuleWriteAccess = useHasProjectAccess({
+ projectId,
+ scope: "evaluationRule:CUD",
+ });
+ const hasAccess = hasEvaluatorReadAccess && hasEvaluationRuleWriteAccess;
const evalTemplates = api.evals.latestTemplates.useQuery(
{
diff --git a/web/src/features/evals/pages/new-template.tsx b/web/src/features/evals/pages/new-template.tsx
index 0c096bcf33..ebcd81f71a 100644
--- a/web/src/features/evals/pages/new-template.tsx
+++ b/web/src/features/evals/pages/new-template.tsx
@@ -9,7 +9,7 @@ export default function NewTemplatesPage() {
const hasAccess = useHasProjectAccess({
projectId,
- scope: "evalTemplate:read",
+ scope: "evaluator:CUD",
});
if (!hasAccess) {
diff --git a/web/src/features/evals/pages/templates.tsx b/web/src/features/evals/pages/templates.tsx
index ad4256f9f5..ef821e5964 100644
--- a/web/src/features/evals/pages/templates.tsx
+++ b/web/src/features/evals/pages/templates.tsx
@@ -19,12 +19,12 @@ export default function TemplatesPage() {
const capture = usePostHogClientCapture();
const hasWriteAccess = useHasProjectAccess({
projectId,
- scope: "evalTemplate:CUD",
+ scope: "evaluator:CUD",
});
const hasReadAccess = useHasProjectAccess({
projectId,
- scope: "evalTemplate:read",
+ scope: "evaluator:read",
});
if (!hasReadAccess) {
diff --git a/web/src/features/evals/server/router.ts b/web/src/features/evals/server/router.ts
index a6aa347ee5..cc87706313 100644
--- a/web/src/features/evals/server/router.ts
+++ b/web/src/features/evals/server/router.ts
@@ -287,7 +287,7 @@ export const evalRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
- scope: "evalJob:read",
+ scope: "evaluationRule:read",
});
return env.LANGFUSE_MAX_HISTORIC_EVAL_CREATION_LIMIT;
}),
@@ -297,7 +297,7 @@ export const evalRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
- scope: "evalJob:read",
+ scope: "evaluationRule:read",
});
return new LegacyEvalCompatibilityService(ctx.prisma).counts(
@@ -318,7 +318,7 @@ export const evalRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
- scope: "evalJob:read",
+ scope: "evaluationRule:read",
});
const result = await new LegacyEvalCompatibilityService(
ctx.prisma,
@@ -355,7 +355,7 @@ export const evalRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
- scope: "evalJob:read",
+ scope: "evaluationRule:read",
});
const config = await new LegacyEvalCompatibilityService(
@@ -388,7 +388,7 @@ export const evalRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
- scope: "evalTemplate:read",
+ scope: "evaluator:read",
});
const service = new LegacyEvalCompatibilityService(ctx.prisma);
@@ -416,7 +416,7 @@ export const evalRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
- scope: "evalTemplate:read",
+ scope: "evaluator:read",
});
return new LegacyEvalCompatibilityService(
@@ -435,7 +435,7 @@ export const evalRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
- scope: "evalTemplate:read",
+ scope: "evaluator:read",
});
return new LegacyEvalCompatibilityService(ctx.prisma).getTemplate(
@@ -456,7 +456,7 @@ export const evalRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
- scope: "evalTemplate:read",
+ scope: "evaluator:read",
});
const service = new LegacyEvalCompatibilityService(ctx.prisma);
@@ -491,7 +491,7 @@ export const evalRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
- scope: "evalTemplate:read",
+ scope: "evaluator:read",
});
const latestTemplates = await new LegacyEvalCompatibilityService(
@@ -519,7 +519,7 @@ export const evalRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
- scope: "evalJob:read",
+ scope: "evaluationRule:read",
});
return {
@@ -540,7 +540,7 @@ export const evalRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
- scope: "evalJob:read",
+ scope: "evaluationRule:read",
});
const targetObjects = Array.isArray(input.targetObject)
@@ -562,7 +562,7 @@ export const evalRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
- scope: "evalJob:read",
+ scope: "evaluationRule:read",
});
return {
@@ -578,7 +578,7 @@ export const evalRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
- scope: "evalJob:CUD",
+ scope: "evaluationRule:CUD",
});
assertCanCreateLegacyEvalJob({
@@ -715,7 +715,7 @@ export const evalRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
- scope: "evalJob:CUD",
+ scope: "evaluator:CUD",
});
assertCodeEvalEnabled();
@@ -754,7 +754,7 @@ export const evalRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
- scope: "evalTemplate:CUD",
+ scope: "evaluator:CUD",
});
await validateEvalTemplateCreation(input);
@@ -826,7 +826,7 @@ export const evalRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: projectId,
- scope: "evalJob:CUD",
+ scope: "evaluationRule:CUD",
});
const compatibility = new LegacyEvalCompatibilityService(ctx.prisma);
@@ -895,7 +895,7 @@ export const evalRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: projectId,
- scope: "evalJob:CUD",
+ scope: "evaluationRule:CUD",
});
const compatibility = new LegacyEvalCompatibilityService(ctx.prisma);
@@ -1086,7 +1086,7 @@ export const evalRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: projectId,
- scope: "evalJob:CUD",
+ scope: "evaluationRule:CUD",
});
const compatibility = new LegacyEvalCompatibilityService(ctx.prisma);
@@ -1130,7 +1130,7 @@ export const evalRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: projectId,
- scope: "evalJob:read",
+ scope: "evaluator:read",
});
return new LegacyEvalCompatibilityService(ctx.prisma).getTemplateUsage(
@@ -1145,7 +1145,7 @@ export const evalRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: projectId,
- scope: "evalTemplate:CUD",
+ scope: "evaluator:CUD",
});
const deletedVersions = await new LegacyEvalCompatibilityService(
@@ -1276,7 +1276,7 @@ export const evalRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
- scope: "evalJob:read",
+ scope: "evaluationRule:read",
});
const { configs } = await new LegacyEvalCompatibilityService(
@@ -1308,7 +1308,7 @@ export const evalRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
- scope: "evalJob:read",
+ scope: "evalJobExecution:read",
});
if (input.evaluatorIds.length === 0) {
@@ -1363,7 +1363,7 @@ export const evalRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
- scope: "evalJob:read",
+ scope: "evalJobExecution:read",
});
const costs: Array<{ id: string; totalCost: number }> =
@@ -1399,7 +1399,7 @@ export const evalRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
- scope: "evalJob:read",
+ scope: "evalJobExecution:read",
});
const costs =
diff --git a/web/src/features/evals/v2/components/Rules/EvaluatorRuleRelationships/EvaluatorRuleRelationships.tsx b/web/src/features/evals/v2/components/Rules/EvaluatorRuleRelationships/EvaluatorRuleRelationships.tsx
index 58baedf223..752a887465 100644
--- a/web/src/features/evals/v2/components/Rules/EvaluatorRuleRelationships/EvaluatorRuleRelationships.tsx
+++ b/web/src/features/evals/v2/components/Rules/EvaluatorRuleRelationships/EvaluatorRuleRelationships.tsx
@@ -117,7 +117,7 @@ function EvaluatorRuleRelationshipsSheet({
const isEstimating = activationConfirmation.estimate.status === "estimating";
const hasWriteAccess = useHasProjectAccess({
projectId,
- scope: "evalJob:CUD",
+ scope: "evaluationRule:CUD",
});
const assignments = api.evalsV2.rules.listRulesForEvaluator.useQuery(
{ projectId, evaluatorId },
diff --git a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
index f1689537d4..69b42fafd3 100644
--- a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
+++ b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
@@ -161,7 +161,7 @@ export function EvaluatorSetupPage(
const capture = usePostHogClientCapture();
const canReactivate = useHasProjectAccess({
projectId,
- scope: "evalTemplate:CUD",
+ scope: "evaluator:CUD",
});
const projectDefaultModel = useProjectDefaultModel({
projectId,
diff --git a/web/src/features/evals/v2/pages/EvaluatorsPage.tsx b/web/src/features/evals/v2/pages/EvaluatorsPage.tsx
index 5bb5cd5212..cb3c56961c 100644
--- a/web/src/features/evals/v2/pages/EvaluatorsPage.tsx
+++ b/web/src/features/evals/v2/pages/EvaluatorsPage.tsx
@@ -270,7 +270,7 @@ export default function EvaluatorsPage() {
filterState.length === 0;
const hasExecutionReadAccess = useHasProjectAccess({
projectId,
- scope: "evalJob:read",
+ scope: "evalJobExecution:read",
});
const defaultModelConnection = projectDefaultModel.connections.find(
({ provider }) => provider === projectDefaultModel.defaultModel?.provider,
diff --git a/web/src/features/evals/v2/pages/RulesPage.tsx b/web/src/features/evals/v2/pages/RulesPage.tsx
index 894a382fd2..f6a979d335 100644
--- a/web/src/features/evals/v2/pages/RulesPage.tsx
+++ b/web/src/features/evals/v2/pages/RulesPage.tsx
@@ -19,11 +19,11 @@ export function RulesPage() {
const [createOpen, setCreateOpen] = useState(false);
const hasReadAccess = useHasProjectAccess({
projectId,
- scope: "evalJob:read",
+ scope: "evaluationRule:read",
});
const hasWriteAccess = useHasProjectAccess({
projectId,
- scope: "evalJob:CUD",
+ scope: "evaluationRule:CUD",
});
if (!hasReadAccess) return <SupportOrUpgradePage />;
diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorRouter.ts b/web/src/features/evals/v2/server/evaluators/evaluatorRouter.ts
index 9632a5afd1..ae9afa4115 100644
--- a/web/src/features/evals/v2/server/evaluators/evaluatorRouter.ts
+++ b/web/src/features/evals/v2/server/evaluators/evaluatorRouter.ts
@@ -59,7 +59,7 @@ export const evaluatorRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: ctx.session.projectId,
- scope: "evalTemplate:read",
+ scope: "evaluator:read",
});
return serviceForContext(ctx).list({
...input,
@@ -73,7 +73,7 @@ export const evaluatorRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: ctx.session.projectId,
- scope: "evalTemplate:read",
+ scope: "evaluator:read",
});
return serviceForContext(ctx).listFilterOptions(ctx.session.projectId);
}),
@@ -84,7 +84,7 @@ export const evaluatorRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: ctx.session.projectId,
- scope: "evalTemplate:read",
+ scope: "evaluator:read",
});
return serviceForContext(ctx).listOptions({
...input,
@@ -98,7 +98,7 @@ export const evaluatorRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: ctx.session.projectId,
- scope: "evalTemplate:read",
+ scope: "evaluator:read",
});
return serviceForContext(ctx).getWithSampleFilter(
ctx.session.projectId,
@@ -112,7 +112,7 @@ export const evaluatorRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: ctx.session.projectId,
- scope: "evalTemplate:read",
+ scope: "evaluator:read",
});
return serviceForContext(ctx).listVersions({
...input,
@@ -126,7 +126,7 @@ export const evaluatorRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: ctx.session.projectId,
- scope: "evalJob:read",
+ scope: "evalJobExecution:read",
});
return serviceForContext(ctx).listRecent({
...input,
@@ -140,7 +140,7 @@ export const evaluatorRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: ctx.session.projectId,
- scope: "evalJob:read",
+ scope: "evalJobExecution:read",
});
return serviceForContext(ctx).getTotalCosts({
...input,
@@ -154,7 +154,7 @@ export const evaluatorRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: ctx.session.projectId,
- scope: "evalJob:CUD",
+ scope: "evaluationRule:CUD",
});
return getActivationCostEstimates({
...input,
@@ -171,7 +171,7 @@ export const evaluatorRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: ctx.session.projectId,
- scope: "evalTemplate:CUD",
+ scope: "evaluator:CUD",
});
const service = serviceForContext(ctx);
return service.create(
@@ -186,7 +186,7 @@ export const evaluatorRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: ctx.session.projectId,
- scope: "evalTemplate:CUD",
+ scope: "evaluator:CUD",
});
const service = serviceForContext(ctx);
return service.update(
@@ -201,7 +201,7 @@ export const evaluatorRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: ctx.session.projectId,
- scope: "evalTemplate:CUD",
+ scope: "evaluator:CUD",
});
return serviceForContext(ctx).reactivate({
projectId: ctx.session.projectId,
@@ -215,7 +215,7 @@ export const evaluatorRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: ctx.session.projectId,
- scope: "evalTemplate:CUD",
+ scope: "evaluator:CUD",
});
const service = serviceForContext(ctx);
await service.delete(ctx.session.projectId, input.evaluatorId);
@@ -228,7 +228,7 @@ export const evaluatorRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: ctx.session.projectId,
- scope: "evalTemplate:CUD",
+ scope: "evaluator:CUD",
});
const service = serviceForContext(ctx);
await service.deleteMany({
@@ -244,7 +244,7 @@ export const evaluatorRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: ctx.session.projectId,
- scope: "evalTemplate:CUD",
+ scope: "evaluator:CUD",
});
return serviceForContext(ctx).testEvaluator({
orgId: ctx.session.orgId,
@@ -265,7 +265,7 @@ export const evaluatorRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: ctx.session.projectId,
- scope: "evalTemplate:CUD",
+ scope: "evaluator:CUD",
});
return serviceForContext(ctx).suggestName({
...input,
@@ -280,7 +280,7 @@ export const evaluatorRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: ctx.session.projectId,
- scope: "evalTemplate:CUD",
+ scope: "evaluator:CUD",
});
return serviceForContext(ctx).suggestDescription({
...input,
diff --git a/web/src/features/evals/v2/server/rules/ruleRouter.ts b/web/src/features/evals/v2/server/rules/ruleRouter.ts
index 2086258f2a..2d0870a226 100644
--- a/web/src/features/evals/v2/server/rules/ruleRouter.ts
+++ b/web/src/features/evals/v2/server/rules/ruleRouter.ts
@@ -41,7 +41,7 @@ export const ruleRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: ctx.session.projectId,
- scope: "evalJob:read",
+ scope: "evaluationRule:read",
});
return serviceForContext(ctx).list({
...input,
@@ -55,7 +55,7 @@ export const ruleRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: ctx.session.projectId,
- scope: "evalJob:read",
+ scope: "evaluationRule:read",
});
return serviceForContext(ctx).listFilterOptions(ctx.session.projectId);
}),
@@ -66,7 +66,7 @@ export const ruleRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: ctx.session.projectId,
- scope: "evalJob:read",
+ scope: "evaluationRule:read",
});
return serviceForContext(ctx).listReusableFilters(ctx.session.projectId);
}),
@@ -75,7 +75,7 @@ export const ruleRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: ctx.session.projectId,
- scope: "evalJob:read",
+ scope: "evaluationRule:read",
});
return serviceForContext(ctx).get(ctx.session.projectId, input.ruleId);
}),
@@ -86,7 +86,7 @@ export const ruleRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: ctx.session.projectId,
- scope: "evalJob:read",
+ scope: "evalJobExecution:read",
});
return serviceForContext(ctx).listRecent({
...input,
@@ -100,7 +100,7 @@ export const ruleRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: ctx.session.projectId,
- scope: "evalJob:read",
+ scope: "evalJobExecution:read",
});
return serviceForContext(ctx).getTotalCosts({
...input,
@@ -114,7 +114,7 @@ export const ruleRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: ctx.session.projectId,
- scope: "evalJob:read",
+ scope: "evaluationRule:read",
});
return serviceForContext(ctx).listRulesForEvaluator(
ctx.session.projectId,
@@ -128,7 +128,7 @@ export const ruleRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: ctx.session.projectId,
- scope: "evalJob:CUD",
+ scope: "evaluationRule:CUD",
});
return serviceForContext(ctx).suggestName({
...input,
@@ -142,7 +142,7 @@ export const ruleRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: ctx.session.projectId,
- scope: "evalJob:CUD",
+ scope: "evaluationRule:CUD",
});
return serviceForContext(ctx).create(
{ ...input, projectId: ctx.session.projectId },
@@ -156,7 +156,7 @@ export const ruleRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: ctx.session.projectId,
- scope: "evalJob:CUD",
+ scope: "evaluationRule:CUD",
});
return serviceForContext(ctx).createOrAttachFromEvaluatorFilters(
{ ...input, projectId: ctx.session.projectId },
@@ -170,7 +170,7 @@ export const ruleRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: ctx.session.projectId,
- scope: "evalJob:CUD",
+ scope: "evaluationRule:CUD",
});
return serviceForContext(ctx).update({
...input,
@@ -184,7 +184,7 @@ export const ruleRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: ctx.session.projectId,
- scope: "evalJob:CUD",
+ scope: "evaluationRule:CUD",
});
return serviceForContext(ctx).setEnabled({
...input,
@@ -198,7 +198,7 @@ export const ruleRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: ctx.session.projectId,
- scope: "evalJob:CUD",
+ scope: "evaluationRule:CUD",
});
await serviceForContext(ctx).delete(ctx.session.projectId, input.ruleId);
return { success: true };
@@ -210,7 +210,7 @@ export const ruleRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: ctx.session.projectId,
- scope: "evalJob:CUD",
+ scope: "evaluationRule:CUD",
});
const ruleIds = await serviceForContext(ctx).deleteMany({
...input,
@@ -225,7 +225,7 @@ export const ruleRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: ctx.session.projectId,
- scope: "evalJob:CUD",
+ scope: "evaluationRule:CUD",
});
const ruleIds = await serviceForContext(ctx).setManyEnabled({
...input,
@@ -240,7 +240,7 @@ export const ruleRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: ctx.session.projectId,
- scope: "evalJob:CUD",
+ scope: "evaluationRule:CUD",
});
return serviceForContext(ctx).attach({
projectId: ctx.session.projectId,
@@ -259,7 +259,7 @@ export const ruleRouter = createTRPCRouter({
throwIfNoProjectAccess({
session: ctx.session,
projectId: ctx.session.projectId,
- scope: "evalJob:CUD",
+ scope: "evaluationRule:CUD",
});
return serviceForContext(ctx).detach({
...input,
diff --git a/web/src/features/events/components/EventsTable.tsx b/web/src/features/events/components/EventsTable.tsx
index 0d982fe0e8..17bc015c31 100644
--- a/web/src/features/events/components/EventsTable.tsx
+++ b/web/src/features/events/components/EventsTable.tsx
@@ -1109,7 +1109,7 @@ export default function ObservationsEventsTable({
disabled: isSelectAllCountUnavailable,
disabledReason: selectAllCountUnavailableReason,
accessCheck: {
- scope: "evalJob:CUD",
+ scope: "evaluationRule:CUD",
},
},
];
diff --git a/web/src/features/experiments/components/MultiStepExperimentForm.tsx b/web/src/features/experiments/components/MultiStepExperimentForm.tsx
index 0ac7179807..c18ad86602 100644
--- a/web/src/features/experiments/components/MultiStepExperimentForm.tsx
+++ b/web/src/features/experiments/components/MultiStepExperimentForm.tsx
@@ -141,12 +141,12 @@ export const MultiStepExperimentForm = ({
const hasEvalReadAccess = useHasProjectAccess({
projectId,
- scope: "evalJob:read",
+ scope: "evaluationRule:read",
});
const hasEvaluatorReadAccess = useHasProjectAccess({
projectId,
- scope: "evalTemplate:read",
+ scope: "evaluator:read",
});
const canReadEvaluators = useV2Evaluators
@@ -155,7 +155,7 @@ export const MultiStepExperimentForm = ({
const hasEvalWriteAccess = useHasProjectAccess({
projectId,
- scope: "evalJob:CUD",
+ scope: "evaluationRule:CUD",
});
const form = useForm({
diff --git a/web/src/features/experiments/components/table/ExperimentItemsTable.tsx b/web/src/features/experiments/components/table/ExperimentItemsTable.tsx
index 346bbac673..d519ba59d5 100644
--- a/web/src/features/experiments/components/table/ExperimentItemsTable.tsx
+++ b/web/src/features/experiments/components/table/ExperimentItemsTable.tsx
@@ -281,7 +281,7 @@ export default function ExperimentItemsTable({
const [showRunEvaluationDialog, setShowRunEvaluationDialog] = useState(false);
const hasEvalAccess = useHasProjectAccess({
projectId,
- scope: "evalJob:CUD",
+ scope: "evaluationRule:CUD",
});
const {
@@ -1085,7 +1085,7 @@ export default function ExperimentItemsTable({
icon: <LightbulbIcon className="h-4 w-4 sm:mr-2" />,
customDialog: true,
accessCheck: {
- scope: "evalJob:CUD",
+ scope: "evaluationRule:CUD",
},
} as TableAction,
]
diff --git a/web/src/features/experiments/components/table/ExperimentsTable.tsx b/web/src/features/experiments/components/table/ExperimentsTable.tsx
index 71cf0ea886..24b49c24a2 100644
--- a/web/src/features/experiments/components/table/ExperimentsTable.tsx
+++ b/web/src/features/experiments/components/table/ExperimentsTable.tsx
@@ -131,7 +131,7 @@ function ExperimentsMultiSelectActionMenu({
const hasEvalAccess = useHasProjectAccess({
projectId,
- scope: "evalJob:CUD",
+ scope: "evaluationRule:CUD",
});
// Build query with experiment context filter for batch actions
@@ -225,7 +225,7 @@ function ExperimentsMultiSelectActionMenu({
icon: <LightbulbIcon className="h-4 w-4 sm:mr-2" />,
customDialog: true,
accessCheck: {
- scope: "evalJob:CUD",
+ scope: "evaluationRule:CUD",
},
} as TableAction,
]
diff --git a/web/src/pages/project/[projectId]/datasets/[datasetId]/experiments/index.tsx b/web/src/pages/project/[projectId]/datasets/[datasetId]/experiments/index.tsx
index afc1e2868d..d56e397815 100644
--- a/web/src/pages/project/[projectId]/datasets/[datasetId]/experiments/index.tsx
+++ b/web/src/pages/project/[projectId]/datasets/[datasetId]/experiments/index.tsx
@@ -110,25 +110,31 @@ export default function Dataset() {
});
};
- const hasEvalReadAccess = useHasProjectAccess({
+ const hasEvaluationRuleReadAccess = useHasProjectAccess({
projectId,
- scope: "evalJob:read",
+ scope: "evaluationRule:read",
});
- const hasEvalWriteAccess = useHasProjectAccess({
+ const hasEvaluationRuleWriteAccess = useHasProjectAccess({
projectId,
- scope: "evalJob:CUD",
+ scope: "evaluationRule:CUD",
+ });
+
+ const hasEvaluatorReadAccess = useHasProjectAccess({
+ projectId,
+ scope: "evaluator:read",
});
const evalTemplates = api.evals.latestTemplates.useQuery(
{ projectId },
- { enabled: !isExperimentsBetaActive },
+ { enabled: !isExperimentsBetaActive && hasEvaluatorReadAccess },
);
const evaluators = api.evals.jobConfigsByTarget.useQuery(
{ projectId, targetObject: ["dataset", "experiment"] },
{
- enabled: !isExperimentsBetaActive && hasEvalReadAccess && !!datasetId,
+ enabled:
+ !isExperimentsBetaActive && hasEvaluationRuleReadAccess && !!datasetId,
},
);
@@ -266,7 +272,7 @@ export default function Dataset() {
</DialogContent>
</Dialog>
- {hasEvalReadAccess && (
+ {hasEvaluationRuleReadAccess && hasEvaluatorReadAccess ? (
<div className="w-fit">
<TemplateSelector
projectId={projectId}
@@ -274,10 +280,10 @@ export default function Dataset() {
evalTemplates={evalTemplates.data?.templates ?? []}
onConfigureTemplate={handleConfigureEvaluator}
onSelectEvaluator={handleSelectEvaluator}
- disabled={!hasEvalWriteAccess}
+ disabled={!hasEvaluationRuleWriteAccess}
/>
</div>
- )}
+ ) : null}
<DatasetAnalytics
key="dataset-analytics"