🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
refactor(evals): rename evaluation RBAC scopes (#16648)
Tobias Wochinger1124345b79bb
diff --git a/packages/shared/src/features/rbac/projectAccessRights.ts b/packages/shared/src/features/rbac/projectAccessRights.tsindex 4fe8643b35..21c6883cbb 100644--- a/packages/shared/src/features/rbac/projectAccessRights.ts+++ b/packages/shared/src/features/rbac/projectAccessRights.ts@@ -44,10 +44,10 @@ export const projectScopes = ["batchExports:create","batchExports:read",- "evalTemplate:CUD",- "evalTemplate:read",- "evalJob:read",- "evalJob:CUD",+ "evaluator:CUD",+ "evaluator:read",+ "evaluationRule:read",+ "evaluationRule:CUD","evalJobExecution:read","evalDefaultModel:read","evalDefaultModel:CUD",@@ -109,10 +109,10 @@ export const projectRoleAccessRights: Record<Role, ProjectScope[]> = {"prompts:read","promptProtectedLabels:CUD","models:CUD",- "evalTemplate:CUD",- "evalTemplate:read",- "evalJob:CUD",- "evalJob:read",+ "evaluator:CUD",+ "evaluator:read",+ "evaluationRule:CUD",+ "evaluationRule:read","evalJobExecution:read","evalDefaultModel:CUD","evalDefaultModel:read",@@ -166,10 +166,10 @@ export const projectRoleAccessRights: Record<Role, ProjectScope[]> = {"prompts:read","promptProtectedLabels:CUD","models:CUD",- "evalTemplate:CUD",- "evalTemplate:read",- "evalJob:CUD",- "evalJob:read",+ "evaluator:CUD",+ "evaluator:read",+ "evaluationRule:CUD",+ "evaluationRule:read","evalJobExecution:read","evalDefaultModel:CUD","evalDefaultModel:read",@@ -216,10 +216,10 @@ export const projectRoleAccessRights: Record<Role, ProjectScope[]> = {"datasets:CUD","prompts:CUD","prompts:read",- "evalTemplate:CUD",- "evalTemplate:read",- "evalJob:read",- "evalJob:CUD",+ "evaluator:CUD",+ "evaluator:read",+ "evaluationRule:read",+ "evaluationRule:CUD","evalJobExecution:read","evalDefaultModel:read","evalDefaultModel:CUD",@@ -249,9 +249,9 @@ export const projectRoleAccessRights: Record<Role, ProjectScope[]> = {VIEWER: ["project:read","prompts:read",- "evalTemplate:read",+ "evaluator:read","scoreConfigs:read",- "evalJob:read",+ "evaluationRule:read","evalJobExecution:read","evalDefaultModel:read","datasets:read",diff --git a/packages/shared/src/in-app-agent/server/mcpPolicy.ts b/packages/shared/src/in-app-agent/server/mcpPolicy.tsindex e82c123699..9df9515268 100644--- a/packages/shared/src/in-app-agent/server/mcpPolicy.ts+++ b/packages/shared/src/in-app-agent/server/mcpPolicy.ts@@ -134,55 +134,55 @@ export const IN_APP_AGENT_LANGFUSE_MCP_TOOL_POLICIES = {},listEvaluators: {approval: "auto",- availability: { scope: "evalTemplate:read" },+ availability: { scope: "evaluator:read" },},listManagedEvaluatorTemplates: {approval: "auto",- availability: { scope: "evalTemplate:read" },+ availability: { scope: "evaluator:read" },},getEvaluator: {approval: "auto",- availability: { scope: "evalTemplate:read" },+ availability: { scope: "evaluator:read" },},createEvaluator: {approval: "approval",- availability: { scope: "evalTemplate:CUD" },+ availability: { scope: "evaluator:CUD" },},updateEvaluator: {approval: "approval",- availability: { scope: "evalTemplate:CUD" },+ availability: { scope: "evaluator:CUD" },},deleteEvaluator: {approval: "approval",- availability: { scope: "evalTemplate:CUD" },+ availability: { scope: "evaluator:CUD" },},listEvaluationRules: {approval: "auto",- availability: { scope: "evalJob:read" },+ availability: { scope: "evaluationRule:read" },},getEvaluationRule: {approval: "auto",- availability: { scope: "evalJob:read" },+ availability: { scope: "evaluationRule:read" },},createEvaluationRule: {approval: "approval",- availability: { scope: "evalJob:CUD" },+ availability: { scope: "evaluationRule:CUD" },},updateEvaluationRule: {approval: "approval",- availability: { scope: "evalJob:CUD" },+ availability: { scope: "evaluationRule:CUD" },},attachEvaluatorToEvaluationRule: {approval: "approval",- availability: { scope: "evalJob:CUD" },+ availability: { scope: "evaluationRule:CUD" },},detachEvaluatorFromEvaluationRule: {approval: "approval",- availability: { scope: "evalJob:CUD" },+ availability: { scope: "evaluationRule:CUD" },},deleteEvaluationRule: {approval: "approval",- availability: { scope: "evalJob:CUD" },+ availability: { scope: "evaluationRule:CUD" },},listExperiments: {approval: "auto",diff --git a/web/src/components/deleteButton.tsx b/web/src/components/deleteButton.tsxindex bf1491e981..9a66b534ae 100644--- a/web/src/components/deleteButton.tsx+++ b/web/src/components/deleteButton.tsx@@ -411,7 +411,7 @@ export function DeleteEvalConfigButton(props: DeleteButtonProps) {const {itemId,projectId,- scope = "evalJob:CUD",+ scope = "evaluationRule:CUD",invalidateFunc = () => utils.evals.invalidate(),} = props;diff --git a/web/src/components/layouts/routes.tsx b/web/src/components/layouts/routes.tsxindex 1ff1a9ff45..c861c744ec 100644--- a/web/src/components/layouts/routes.tsx+++ b/web/src/components/layouts/routes.tsx@@ -169,7 +169,7 @@ export const ROUTES: Route[] = [title: "Evaluators",icon: Lightbulb,productModule: "evaluation",- projectRbacScopes: ["evalJob:read"],+ projectRbacScopes: ["evaluator:read", "evaluationRule:read"],group: RouteGroup.Evaluation,section: RouteSection.Main,pathname: `/project/[projectId]/evals`,diff --git a/web/src/components/table/peek/peek-evaluator-config-detail.tsx b/web/src/components/table/peek/peek-evaluator-config-detail.tsxindex 7157ed0866..1c29751fd5 100644--- a/web/src/components/table/peek/peek-evaluator-config-detail.tsx+++ b/web/src/components/table/peek/peek-evaluator-config-detail.tsx@@ -51,7 +51,10 @@ const PeekViewEvaluatorConfigDetail = ({evaluatorId: evalConfig?.id,evaluator: evalConfig,});- const hasAccess = useHasProjectAccess({ projectId, scope: "evalJob:CUD" });+ const hasAccess = useHasProjectAccess({+ projectId,+ scope: "evaluationRule:CUD",+ });if (!evalConfig) {return <Skeleton className="h-full w-full rounded-none" />;diff --git a/web/src/features/batch-actions/server/runEvaluationRouter.ts b/web/src/features/batch-actions/server/runEvaluationRouter.tsindex e82fc12b94..e7927e5730 100644--- a/web/src/features/batch-actions/server/runEvaluationRouter.ts+++ b/web/src/features/batch-actions/server/runEvaluationRouter.ts@@ -32,7 +32,7 @@ export const runEvaluationRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: input.projectId,- scope: "evalJob:CUD",+ scope: "evaluationRule:CUD",});const {diff --git a/web/src/features/evals/components/deactivate-config.tsx b/web/src/features/evals/components/deactivate-config.tsxindex 85be7a5f0b..b0eb34a1b8 100644--- a/web/src/features/evals/components/deactivate-config.tsx+++ b/web/src/features/evals/components/deactivate-config.tsx@@ -25,7 +25,10 @@ export function DeactivateEvalConfig({onStatusChange?: () => void;}) {const utils = api.useUtils();- const hasAccess = useHasProjectAccess({ projectId, scope: "evalJob:CUD" });+ const hasAccess = useHasProjectAccess({+ projectId,+ scope: "evaluationRule:CUD",+ });const { allowLegacy } = useEvalCapabilities(projectId);const [isOpen, setIsOpen] = useState(false);const capture = usePostHogClientCapture();diff --git a/web/src/features/evals/components/eval-template-detail.tsx b/web/src/features/evals/components/eval-template-detail.tsxindex 6621c5d511..ac0f852dbf 100644--- a/web/src/features/evals/components/eval-template-detail.tsx+++ b/web/src/features/evals/components/eval-template-detail.tsx@@ -31,7 +31,7 @@ export const EvalTemplateDetail = () => {const capture = usePostHogClientCapture();const hasDeleteAccess = useHasProjectAccess({projectId,- scope: "evalTemplate:CUD",+ scope: "evaluator:CUD",});// get the current template by id@@ -231,7 +231,7 @@ function UpdateTemplate({}) {const hasAccess = useHasProjectAccess({projectId,- scope: "evalTemplate:CUD",+ scope: "evaluator:CUD",});const capture = usePostHogClientCapture();diff --git a/web/src/features/evals/components/eval-templates-table.tsx b/web/src/features/evals/components/eval-templates-table.tsxindex 0a3585f516..4b1b44dfb1 100644--- a/web/src/features/evals/components/eval-templates-table.tsx+++ b/web/src/features/evals/components/eval-templates-table.tsx@@ -148,7 +148,7 @@ const EvalTemplateRowActionsMenu = ({const utils = api.useUtils();const hasTemplateWriteAccess = useHasProjectAccess({projectId,- scope: "evalTemplate:CUD",+ scope: "evaluator:CUD",});return (@@ -255,7 +255,10 @@ export default function EvalsTemplateTable({searchQuery: searchQuery,});- const hasAccess = useHasProjectAccess({ projectId, scope: "evalJob:CUD" });+ const hasAccess = useHasProjectAccess({+ projectId,+ scope: "evaluator:CUD",+ });const totalCount = templates.data?.totalCount ?? null;diff --git a/web/src/features/evals/components/evaluator-table.tsx b/web/src/features/evals/components/evaluator-table.tsxindex 8418107313..3c17e38cc8 100644--- a/web/src/features/evals/components/evaluator-table.tsx+++ b/web/src/features/evals/components/evaluator-table.tsx@@ -126,7 +126,10 @@ export default function EvaluatorTable({ projectId }: { projectId: string }) {},);- const hasAccess = useHasProjectAccess({ projectId, scope: "evalJob:CUD" });+ const hasAccess = useHasProjectAccess({+ projectId,+ scope: "evaluationRule:CUD",+ });// Deprecated evaluators are read-only where new legacy setups are not// allowed (cloud); self-hosted deployments keep editing them.const { allowLegacy } = useEvalCapabilities(projectId);diff --git a/web/src/features/evals/pages/evaluators.tsx b/web/src/features/evals/pages/evaluators.tsxindex 6d909a4b28..c0efd92680 100644--- a/web/src/features/evals/pages/evaluators.tsx+++ b/web/src/features/evals/pages/evaluators.tsx@@ -24,12 +24,12 @@ export default function EvaluatorsPage() {);const hasWriteAccess = useHasProjectAccess({projectId,- scope: "evalJob:CUD",+ scope: "evaluationRule:CUD",});const hasReadAccess = useHasProjectAccess({projectId,- scope: "evalJob:read",+ scope: "evaluationRule:read",});// Fetch counts of evaluator configs and templatesdiff --git a/web/src/features/evals/pages/new-evaluator.tsx b/web/src/features/evals/pages/new-evaluator.tsxindex bd1800b1e6..cf017fff91 100644--- a/web/src/features/evals/pages/new-evaluator.tsx+++ b/web/src/features/evals/pages/new-evaluator.tsx@@ -51,10 +51,15 @@ export default function NewEvaluatorPage() {const hasDefaultModel =!!defaultModelQuery.data || defaultModelConfiguredInFlow;- const hasAccess = useHasProjectAccess({+ const hasEvaluatorReadAccess = useHasProjectAccess({projectId,- scope: "evalTemplate:CUD",+ scope: "evaluator:read",});+ const hasEvaluationRuleWriteAccess = useHasProjectAccess({+ projectId,+ scope: "evaluationRule:CUD",+ });+ const hasAccess = hasEvaluatorReadAccess && hasEvaluationRuleWriteAccess;const evalTemplates = api.evals.latestTemplates.useQuery({diff --git a/web/src/features/evals/pages/new-template.tsx b/web/src/features/evals/pages/new-template.tsxindex 0c096bcf33..ebcd81f71a 100644--- a/web/src/features/evals/pages/new-template.tsx+++ b/web/src/features/evals/pages/new-template.tsx@@ -9,7 +9,7 @@ export default function NewTemplatesPage() {const hasAccess = useHasProjectAccess({projectId,- scope: "evalTemplate:read",+ scope: "evaluator:CUD",});if (!hasAccess) {diff --git a/web/src/features/evals/pages/templates.tsx b/web/src/features/evals/pages/templates.tsxindex ad4256f9f5..ef821e5964 100644--- a/web/src/features/evals/pages/templates.tsx+++ b/web/src/features/evals/pages/templates.tsx@@ -19,12 +19,12 @@ export default function TemplatesPage() {const capture = usePostHogClientCapture();const hasWriteAccess = useHasProjectAccess({projectId,- scope: "evalTemplate:CUD",+ scope: "evaluator:CUD",});const hasReadAccess = useHasProjectAccess({projectId,- scope: "evalTemplate:read",+ scope: "evaluator:read",});if (!hasReadAccess) {diff --git a/web/src/features/evals/server/router.ts b/web/src/features/evals/server/router.tsindex a6aa347ee5..cc87706313 100644--- a/web/src/features/evals/server/router.ts+++ b/web/src/features/evals/server/router.ts@@ -287,7 +287,7 @@ export const evalRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: input.projectId,- scope: "evalJob:read",+ scope: "evaluationRule:read",});return env.LANGFUSE_MAX_HISTORIC_EVAL_CREATION_LIMIT;}),@@ -297,7 +297,7 @@ export const evalRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: input.projectId,- scope: "evalJob:read",+ scope: "evaluationRule:read",});return new LegacyEvalCompatibilityService(ctx.prisma).counts(@@ -318,7 +318,7 @@ export const evalRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: input.projectId,- scope: "evalJob:read",+ scope: "evaluationRule:read",});const result = await new LegacyEvalCompatibilityService(ctx.prisma,@@ -355,7 +355,7 @@ export const evalRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: input.projectId,- scope: "evalJob:read",+ scope: "evaluationRule:read",});const config = await new LegacyEvalCompatibilityService(@@ -388,7 +388,7 @@ export const evalRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: input.projectId,- scope: "evalTemplate:read",+ scope: "evaluator:read",});const service = new LegacyEvalCompatibilityService(ctx.prisma);@@ -416,7 +416,7 @@ export const evalRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: input.projectId,- scope: "evalTemplate:read",+ scope: "evaluator:read",});return new LegacyEvalCompatibilityService(@@ -435,7 +435,7 @@ export const evalRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: input.projectId,- scope: "evalTemplate:read",+ scope: "evaluator:read",});return new LegacyEvalCompatibilityService(ctx.prisma).getTemplate(@@ -456,7 +456,7 @@ export const evalRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: input.projectId,- scope: "evalTemplate:read",+ scope: "evaluator:read",});const service = new LegacyEvalCompatibilityService(ctx.prisma);@@ -491,7 +491,7 @@ export const evalRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: input.projectId,- scope: "evalTemplate:read",+ scope: "evaluator:read",});const latestTemplates = await new LegacyEvalCompatibilityService(@@ -519,7 +519,7 @@ export const evalRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: input.projectId,- scope: "evalJob:read",+ scope: "evaluationRule:read",});return {@@ -540,7 +540,7 @@ export const evalRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: input.projectId,- scope: "evalJob:read",+ scope: "evaluationRule:read",});const targetObjects = Array.isArray(input.targetObject)@@ -562,7 +562,7 @@ export const evalRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: input.projectId,- scope: "evalJob:read",+ scope: "evaluationRule:read",});return {@@ -578,7 +578,7 @@ export const evalRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: input.projectId,- scope: "evalJob:CUD",+ scope: "evaluationRule:CUD",});assertCanCreateLegacyEvalJob({@@ -715,7 +715,7 @@ export const evalRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: input.projectId,- scope: "evalJob:CUD",+ scope: "evaluator:CUD",});assertCodeEvalEnabled();@@ -754,7 +754,7 @@ export const evalRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: input.projectId,- scope: "evalTemplate:CUD",+ scope: "evaluator:CUD",});await validateEvalTemplateCreation(input);@@ -826,7 +826,7 @@ export const evalRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: projectId,- scope: "evalJob:CUD",+ scope: "evaluationRule:CUD",});const compatibility = new LegacyEvalCompatibilityService(ctx.prisma);@@ -895,7 +895,7 @@ export const evalRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: projectId,- scope: "evalJob:CUD",+ scope: "evaluationRule:CUD",});const compatibility = new LegacyEvalCompatibilityService(ctx.prisma);@@ -1086,7 +1086,7 @@ export const evalRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: projectId,- scope: "evalJob:CUD",+ scope: "evaluationRule:CUD",});const compatibility = new LegacyEvalCompatibilityService(ctx.prisma);@@ -1130,7 +1130,7 @@ export const evalRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: projectId,- scope: "evalJob:read",+ scope: "evaluator:read",});return new LegacyEvalCompatibilityService(ctx.prisma).getTemplateUsage(@@ -1145,7 +1145,7 @@ export const evalRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: projectId,- scope: "evalTemplate:CUD",+ scope: "evaluator:CUD",});const deletedVersions = await new LegacyEvalCompatibilityService(@@ -1276,7 +1276,7 @@ export const evalRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: input.projectId,- scope: "evalJob:read",+ scope: "evaluationRule:read",});const { configs } = await new LegacyEvalCompatibilityService(@@ -1308,7 +1308,7 @@ export const evalRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: input.projectId,- scope: "evalJob:read",+ scope: "evalJobExecution:read",});if (input.evaluatorIds.length === 0) {@@ -1363,7 +1363,7 @@ export const evalRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: input.projectId,- scope: "evalJob:read",+ scope: "evalJobExecution:read",});const costs: Array<{ id: string; totalCost: number }> =@@ -1399,7 +1399,7 @@ export const evalRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: input.projectId,- scope: "evalJob:read",+ scope: "evalJobExecution:read",});const costs =diff --git a/web/src/features/evals/v2/components/Rules/EvaluatorRuleRelationships/EvaluatorRuleRelationships.tsx b/web/src/features/evals/v2/components/Rules/EvaluatorRuleRelationships/EvaluatorRuleRelationships.tsxindex 58baedf223..752a887465 100644--- a/web/src/features/evals/v2/components/Rules/EvaluatorRuleRelationships/EvaluatorRuleRelationships.tsx+++ b/web/src/features/evals/v2/components/Rules/EvaluatorRuleRelationships/EvaluatorRuleRelationships.tsx@@ -117,7 +117,7 @@ function EvaluatorRuleRelationshipsSheet({const isEstimating = activationConfirmation.estimate.status === "estimating";const hasWriteAccess = useHasProjectAccess({projectId,- scope: "evalJob:CUD",+ scope: "evaluationRule:CUD",});const assignments = api.evalsV2.rules.listRulesForEvaluator.useQuery({ projectId, evaluatorId },diff --git a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsxindex f1689537d4..69b42fafd3 100644--- a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx+++ b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx@@ -161,7 +161,7 @@ export function EvaluatorSetupPage(const capture = usePostHogClientCapture();const canReactivate = useHasProjectAccess({projectId,- scope: "evalTemplate:CUD",+ scope: "evaluator:CUD",});const projectDefaultModel = useProjectDefaultModel({projectId,diff --git a/web/src/features/evals/v2/pages/EvaluatorsPage.tsx b/web/src/features/evals/v2/pages/EvaluatorsPage.tsxindex 5bb5cd5212..cb3c56961c 100644--- a/web/src/features/evals/v2/pages/EvaluatorsPage.tsx+++ b/web/src/features/evals/v2/pages/EvaluatorsPage.tsx@@ -270,7 +270,7 @@ export default function EvaluatorsPage() {filterState.length === 0;const hasExecutionReadAccess = useHasProjectAccess({projectId,- scope: "evalJob:read",+ scope: "evalJobExecution:read",});const defaultModelConnection = projectDefaultModel.connections.find(({ provider }) => provider === projectDefaultModel.defaultModel?.provider,diff --git a/web/src/features/evals/v2/pages/RulesPage.tsx b/web/src/features/evals/v2/pages/RulesPage.tsxindex 894a382fd2..f6a979d335 100644--- a/web/src/features/evals/v2/pages/RulesPage.tsx+++ b/web/src/features/evals/v2/pages/RulesPage.tsx@@ -19,11 +19,11 @@ export function RulesPage() {const [createOpen, setCreateOpen] = useState(false);const hasReadAccess = useHasProjectAccess({projectId,- scope: "evalJob:read",+ scope: "evaluationRule:read",});const hasWriteAccess = useHasProjectAccess({projectId,- scope: "evalJob:CUD",+ scope: "evaluationRule:CUD",});if (!hasReadAccess) return <SupportOrUpgradePage />;diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorRouter.ts b/web/src/features/evals/v2/server/evaluators/evaluatorRouter.tsindex 9632a5afd1..ae9afa4115 100644--- a/web/src/features/evals/v2/server/evaluators/evaluatorRouter.ts+++ b/web/src/features/evals/v2/server/evaluators/evaluatorRouter.ts@@ -59,7 +59,7 @@ export const evaluatorRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: ctx.session.projectId,- scope: "evalTemplate:read",+ scope: "evaluator:read",});return serviceForContext(ctx).list({...input,@@ -73,7 +73,7 @@ export const evaluatorRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: ctx.session.projectId,- scope: "evalTemplate:read",+ scope: "evaluator:read",});return serviceForContext(ctx).listFilterOptions(ctx.session.projectId);}),@@ -84,7 +84,7 @@ export const evaluatorRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: ctx.session.projectId,- scope: "evalTemplate:read",+ scope: "evaluator:read",});return serviceForContext(ctx).listOptions({...input,@@ -98,7 +98,7 @@ export const evaluatorRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: ctx.session.projectId,- scope: "evalTemplate:read",+ scope: "evaluator:read",});return serviceForContext(ctx).getWithSampleFilter(ctx.session.projectId,@@ -112,7 +112,7 @@ export const evaluatorRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: ctx.session.projectId,- scope: "evalTemplate:read",+ scope: "evaluator:read",});return serviceForContext(ctx).listVersions({...input,@@ -126,7 +126,7 @@ export const evaluatorRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: ctx.session.projectId,- scope: "evalJob:read",+ scope: "evalJobExecution:read",});return serviceForContext(ctx).listRecent({...input,@@ -140,7 +140,7 @@ export const evaluatorRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: ctx.session.projectId,- scope: "evalJob:read",+ scope: "evalJobExecution:read",});return serviceForContext(ctx).getTotalCosts({...input,@@ -154,7 +154,7 @@ export const evaluatorRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: ctx.session.projectId,- scope: "evalJob:CUD",+ scope: "evaluationRule:CUD",});return getActivationCostEstimates({...input,@@ -171,7 +171,7 @@ export const evaluatorRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: ctx.session.projectId,- scope: "evalTemplate:CUD",+ scope: "evaluator:CUD",});const service = serviceForContext(ctx);return service.create(@@ -186,7 +186,7 @@ export const evaluatorRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: ctx.session.projectId,- scope: "evalTemplate:CUD",+ scope: "evaluator:CUD",});const service = serviceForContext(ctx);return service.update(@@ -201,7 +201,7 @@ export const evaluatorRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: ctx.session.projectId,- scope: "evalTemplate:CUD",+ scope: "evaluator:CUD",});return serviceForContext(ctx).reactivate({projectId: ctx.session.projectId,@@ -215,7 +215,7 @@ export const evaluatorRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: ctx.session.projectId,- scope: "evalTemplate:CUD",+ scope: "evaluator:CUD",});const service = serviceForContext(ctx);await service.delete(ctx.session.projectId, input.evaluatorId);@@ -228,7 +228,7 @@ export const evaluatorRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: ctx.session.projectId,- scope: "evalTemplate:CUD",+ scope: "evaluator:CUD",});const service = serviceForContext(ctx);await service.deleteMany({@@ -244,7 +244,7 @@ export const evaluatorRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: ctx.session.projectId,- scope: "evalTemplate:CUD",+ scope: "evaluator:CUD",});return serviceForContext(ctx).testEvaluator({orgId: ctx.session.orgId,@@ -265,7 +265,7 @@ export const evaluatorRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: ctx.session.projectId,- scope: "evalTemplate:CUD",+ scope: "evaluator:CUD",});return serviceForContext(ctx).suggestName({...input,@@ -280,7 +280,7 @@ export const evaluatorRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: ctx.session.projectId,- scope: "evalTemplate:CUD",+ scope: "evaluator:CUD",});return serviceForContext(ctx).suggestDescription({...input,diff --git a/web/src/features/evals/v2/server/rules/ruleRouter.ts b/web/src/features/evals/v2/server/rules/ruleRouter.tsindex 2086258f2a..2d0870a226 100644--- a/web/src/features/evals/v2/server/rules/ruleRouter.ts+++ b/web/src/features/evals/v2/server/rules/ruleRouter.ts@@ -41,7 +41,7 @@ export const ruleRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: ctx.session.projectId,- scope: "evalJob:read",+ scope: "evaluationRule:read",});return serviceForContext(ctx).list({...input,@@ -55,7 +55,7 @@ export const ruleRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: ctx.session.projectId,- scope: "evalJob:read",+ scope: "evaluationRule:read",});return serviceForContext(ctx).listFilterOptions(ctx.session.projectId);}),@@ -66,7 +66,7 @@ export const ruleRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: ctx.session.projectId,- scope: "evalJob:read",+ scope: "evaluationRule:read",});return serviceForContext(ctx).listReusableFilters(ctx.session.projectId);}),@@ -75,7 +75,7 @@ export const ruleRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: ctx.session.projectId,- scope: "evalJob:read",+ scope: "evaluationRule:read",});return serviceForContext(ctx).get(ctx.session.projectId, input.ruleId);}),@@ -86,7 +86,7 @@ export const ruleRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: ctx.session.projectId,- scope: "evalJob:read",+ scope: "evalJobExecution:read",});return serviceForContext(ctx).listRecent({...input,@@ -100,7 +100,7 @@ export const ruleRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: ctx.session.projectId,- scope: "evalJob:read",+ scope: "evalJobExecution:read",});return serviceForContext(ctx).getTotalCosts({...input,@@ -114,7 +114,7 @@ export const ruleRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: ctx.session.projectId,- scope: "evalJob:read",+ scope: "evaluationRule:read",});return serviceForContext(ctx).listRulesForEvaluator(ctx.session.projectId,@@ -128,7 +128,7 @@ export const ruleRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: ctx.session.projectId,- scope: "evalJob:CUD",+ scope: "evaluationRule:CUD",});return serviceForContext(ctx).suggestName({...input,@@ -142,7 +142,7 @@ export const ruleRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: ctx.session.projectId,- scope: "evalJob:CUD",+ scope: "evaluationRule:CUD",});return serviceForContext(ctx).create({ ...input, projectId: ctx.session.projectId },@@ -156,7 +156,7 @@ export const ruleRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: ctx.session.projectId,- scope: "evalJob:CUD",+ scope: "evaluationRule:CUD",});return serviceForContext(ctx).createOrAttachFromEvaluatorFilters({ ...input, projectId: ctx.session.projectId },@@ -170,7 +170,7 @@ export const ruleRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: ctx.session.projectId,- scope: "evalJob:CUD",+ scope: "evaluationRule:CUD",});return serviceForContext(ctx).update({...input,@@ -184,7 +184,7 @@ export const ruleRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: ctx.session.projectId,- scope: "evalJob:CUD",+ scope: "evaluationRule:CUD",});return serviceForContext(ctx).setEnabled({...input,@@ -198,7 +198,7 @@ export const ruleRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: ctx.session.projectId,- scope: "evalJob:CUD",+ scope: "evaluationRule:CUD",});await serviceForContext(ctx).delete(ctx.session.projectId, input.ruleId);return { success: true };@@ -210,7 +210,7 @@ export const ruleRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: ctx.session.projectId,- scope: "evalJob:CUD",+ scope: "evaluationRule:CUD",});const ruleIds = await serviceForContext(ctx).deleteMany({...input,@@ -225,7 +225,7 @@ export const ruleRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: ctx.session.projectId,- scope: "evalJob:CUD",+ scope: "evaluationRule:CUD",});const ruleIds = await serviceForContext(ctx).setManyEnabled({...input,@@ -240,7 +240,7 @@ export const ruleRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: ctx.session.projectId,- scope: "evalJob:CUD",+ scope: "evaluationRule:CUD",});return serviceForContext(ctx).attach({projectId: ctx.session.projectId,@@ -259,7 +259,7 @@ export const ruleRouter = createTRPCRouter({throwIfNoProjectAccess({session: ctx.session,projectId: ctx.session.projectId,- scope: "evalJob:CUD",+ scope: "evaluationRule:CUD",});return serviceForContext(ctx).detach({...input,diff --git a/web/src/features/events/components/EventsTable.tsx b/web/src/features/events/components/EventsTable.tsxindex 0d982fe0e8..17bc015c31 100644--- a/web/src/features/events/components/EventsTable.tsx+++ b/web/src/features/events/components/EventsTable.tsx@@ -1109,7 +1109,7 @@ export default function ObservationsEventsTable({disabled: isSelectAllCountUnavailable,disabledReason: selectAllCountUnavailableReason,accessCheck: {- scope: "evalJob:CUD",+ scope: "evaluationRule:CUD",},},];diff --git a/web/src/features/experiments/components/MultiStepExperimentForm.tsx b/web/src/features/experiments/components/MultiStepExperimentForm.tsxindex 0ac7179807..c18ad86602 100644--- a/web/src/features/experiments/components/MultiStepExperimentForm.tsx+++ b/web/src/features/experiments/components/MultiStepExperimentForm.tsx@@ -141,12 +141,12 @@ export const MultiStepExperimentForm = ({const hasEvalReadAccess = useHasProjectAccess({projectId,- scope: "evalJob:read",+ scope: "evaluationRule:read",});const hasEvaluatorReadAccess = useHasProjectAccess({projectId,- scope: "evalTemplate:read",+ scope: "evaluator:read",});const canReadEvaluators = useV2Evaluators@@ -155,7 +155,7 @@ export const MultiStepExperimentForm = ({const hasEvalWriteAccess = useHasProjectAccess({projectId,- scope: "evalJob:CUD",+ scope: "evaluationRule:CUD",});const form = useForm({diff --git a/web/src/features/experiments/components/table/ExperimentItemsTable.tsx b/web/src/features/experiments/components/table/ExperimentItemsTable.tsxindex 346bbac673..d519ba59d5 100644--- a/web/src/features/experiments/components/table/ExperimentItemsTable.tsx+++ b/web/src/features/experiments/components/table/ExperimentItemsTable.tsx@@ -281,7 +281,7 @@ export default function ExperimentItemsTable({const [showRunEvaluationDialog, setShowRunEvaluationDialog] = useState(false);const hasEvalAccess = useHasProjectAccess({projectId,- scope: "evalJob:CUD",+ scope: "evaluationRule:CUD",});const {@@ -1085,7 +1085,7 @@ export default function ExperimentItemsTable({icon: <LightbulbIcon className="h-4 w-4 sm:mr-2" />,customDialog: true,accessCheck: {- scope: "evalJob:CUD",+ scope: "evaluationRule:CUD",},} as TableAction,]diff --git a/web/src/features/experiments/components/table/ExperimentsTable.tsx b/web/src/features/experiments/components/table/ExperimentsTable.tsxindex 71cf0ea886..24b49c24a2 100644--- a/web/src/features/experiments/components/table/ExperimentsTable.tsx+++ b/web/src/features/experiments/components/table/ExperimentsTable.tsx@@ -131,7 +131,7 @@ function ExperimentsMultiSelectActionMenu({const hasEvalAccess = useHasProjectAccess({projectId,- scope: "evalJob:CUD",+ scope: "evaluationRule:CUD",});// Build query with experiment context filter for batch actions@@ -225,7 +225,7 @@ function ExperimentsMultiSelectActionMenu({icon: <LightbulbIcon className="h-4 w-4 sm:mr-2" />,customDialog: true,accessCheck: {- scope: "evalJob:CUD",+ scope: "evaluationRule:CUD",},} as TableAction,]diff --git a/web/src/pages/project/[projectId]/datasets/[datasetId]/experiments/index.tsx b/web/src/pages/project/[projectId]/datasets/[datasetId]/experiments/index.tsxindex afc1e2868d..d56e397815 100644--- a/web/src/pages/project/[projectId]/datasets/[datasetId]/experiments/index.tsx+++ b/web/src/pages/project/[projectId]/datasets/[datasetId]/experiments/index.tsx@@ -110,25 +110,31 @@ export default function Dataset() {});};- const hasEvalReadAccess = useHasProjectAccess({+ const hasEvaluationRuleReadAccess = useHasProjectAccess({projectId,- scope: "evalJob:read",+ scope: "evaluationRule:read",});- const hasEvalWriteAccess = useHasProjectAccess({+ const hasEvaluationRuleWriteAccess = useHasProjectAccess({projectId,- scope: "evalJob:CUD",+ scope: "evaluationRule:CUD",+ });++ const hasEvaluatorReadAccess = useHasProjectAccess({+ projectId,+ scope: "evaluator:read",});const evalTemplates = api.evals.latestTemplates.useQuery({ projectId },- { enabled: !isExperimentsBetaActive },+ { enabled: !isExperimentsBetaActive && hasEvaluatorReadAccess },);const evaluators = api.evals.jobConfigsByTarget.useQuery({ projectId, targetObject: ["dataset", "experiment"] },{- enabled: !isExperimentsBetaActive && hasEvalReadAccess && !!datasetId,+ enabled:+ !isExperimentsBetaActive && hasEvaluationRuleReadAccess && !!datasetId,},);@@ -266,7 +272,7 @@ export default function Dataset() {</DialogContent></Dialog>- {hasEvalReadAccess && (+ {hasEvaluationRuleReadAccess && hasEvaluatorReadAccess ? (<div className="w-fit"><TemplateSelectorprojectId={projectId}@@ -274,10 +280,10 @@ export default function Dataset() {evalTemplates={evalTemplates.data?.templates ?? []}onConfigureTemplate={handleConfigureEvaluator}onSelectEvaluator={handleSelectEvaluator}- disabled={!hasEvalWriteAccess}+ disabled={!hasEvaluationRuleWriteAccess}/></div>- )}+ ) : null}<DatasetAnalyticskey="dataset-analytics"