🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(evals): activate disabled rules on evaluator attach (#16465)
Tobias Wochinger81b1e751ae19
diff --git a/web/src/__tests__/server/evaluation-rule-v2-service.servertest.ts b/web/src/__tests__/server/evaluation-rule-v2-service.servertest.tsindex 255b128946..0644077bb5 100644--- a/web/src/__tests__/server/evaluation-rule-v2-service.servertest.ts+++ b/web/src/__tests__/server/evaluation-rule-v2-service.servertest.ts@@ -717,6 +717,26 @@ describe("RuleService", () => {});});+ it("enables a disabled rule when attaching with activation", async () => {+ const evaluator = await createEvaluator();+ const service = createService();+ const rule = await service.create(+ { ...createInput(), enabled: false },+ null,+ );+ const attachment = {+ projectId,+ ruleId: rule.id,+ assignment: { evaluatorId: evaluator.id, variableMapping: null },+ enableRule: true,+ };++ await expect(service.attach(attachment)).resolves.toMatchObject({+ enabled: true,+ assignments: [expect.objectContaining({ evaluatorId: evaluator.id })],+ });+ });+it("updates experiment filters and assignments", async () => {const [first, second] = await Promise.all([createEvaluator(),diff --git a/web/src/__tests__/server/evaluation-rule-v2-trpc.servertest.ts b/web/src/__tests__/server/evaluation-rule-v2-trpc.servertest.tsindex cc828a40fa..b2f6ac1c90 100644--- a/web/src/__tests__/server/evaluation-rule-v2-trpc.servertest.ts+++ b/web/src/__tests__/server/evaluation-rule-v2-trpc.servertest.ts@@ -252,12 +252,15 @@ describe("evaluation rule v2 tRPC", () => {).resolves.toBe(0);});- it("supports attaching and detaching an evaluator", async () => {+ it("supports attaching with activation and detaching an evaluator", async () => {const [first, second] = await Promise.all([createEvaluator("First transport evaluator"),createEvaluator("Second transport evaluator"),]);- const rule = await caller.evalsV2.rules.create(ruleInput(first.id));+ const rule = await caller.evalsV2.rules.create({+ ...ruleInput(first.id),+ enabled: false,+ });const attached = await caller.evalsV2.rules.attach({projectId,ruleId: rule.id,@@ -265,8 +268,10 @@ describe("evaluation rule v2 tRPC", () => {variableMapping: [{ templateVariable: "output", selectedColumnId: "input" },],+ enableRule: true,});+ expect(attached.enabled).toBe(true);expect(attached.assignments).toEqual(expect.arrayContaining([expect.objectContaining({ evaluatorId: second.id }),diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsxindex 1935c06a23..33eb0a3efb 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsx@@ -69,7 +69,7 @@ export function EvaluatorSavedDialogContainer({evaluator.hasCompletedTestCall ?? false,);const missingCostTestRequest = useRef<Promise<void> | null>(null);- const rules = api.evalsV2.rules.list.useQuery(+ const activeRules = api.evalsV2.rules.list.useQuery({projectId,page: 1,@@ -79,13 +79,27 @@ export function EvaluatorSavedDialogContainer({},{ enabled: dialogPhase === "saved" },);+ const inactiveRules = api.evalsV2.rules.list.useQuery(+ {+ projectId,+ page: 1,+ limit: 100,+ enabled: false,+ targetObjects: [EvalTargetObject.EVENT, EvalTargetObject.EXPERIMENT],+ },+ { enabled: dialogPhase === "saved" },+ );const availableRules = useMemo(() =>- [...(rules.data?.rules ?? [])].sort(+ [+ ...(activeRules.data?.rules ?? []),+ ...(inactiveRules.data?.rules ?? []),+ ].sort((left, right) => right.assignments.length - left.assignments.length,),- [rules.data?.rules],+ [activeRules.data?.rules, inactiveRules.data?.rules],);+ const rulesPending = activeRules.isPending || inactiveRules.isPending;const {supportedFilters: supportedRuleFilters,unsupportedReasons: unsupportedRuleFilterReasons,@@ -126,6 +140,7 @@ export function EvaluatorSavedDialogContainer({ruleId: rule.id,evaluatorId: evaluator.id,variableMapping: null,+ enableRule: !rule.enabled,});capture("evaluation_rules:attach_evaluator", {evaluatorCount: 1,@@ -285,7 +300,7 @@ export function EvaluatorSavedDialogContainer({if (mode !== "different-scope" ||selectedRuleId !== undefined ||- rules.isPending+ rulesPending) {return;}@@ -296,13 +311,7 @@ export function EvaluatorSavedDialogContainer({} else {setSelectedRuleId(null);}- }, [- availableRules,- mode,- rules.isPending,- selectExistingRule,- selectedRuleId,- ]);+ }, [availableRules, mode, rulesPending, selectExistingRule, selectedRuleId]);const handleModeChange = (nextMode: EvaluatorSavedMode) => {estimateRequestId.current += 1;@@ -314,7 +323,7 @@ export function EvaluatorSavedDialogContainer({return;}setSelectedRuleId(- nextMode === "different-scope" && !rules.isPending ? null : undefined,+ nextMode === "different-scope" && !rulesPending ? null : undefined,);setIsEstimating(false);if (nextMode === "test-filters") {@@ -349,7 +358,7 @@ export function EvaluatorSavedDialogContainer({onOpenChange={setRulePickerOpen}disabledRules={[]}availableRules={availableRules}- loading={rules.isPending}+ loading={rulesPending}onSelectAvailableRule={selectExistingRule}onCreateRule={selectNewRule}>@@ -395,7 +404,7 @@ export function EvaluatorSavedDialogContainer({/>{selectedRule.assignments.length > 0 ? (<p className="text-muted-foreground text-sm">- Already running on this rule:{" "}+ Already attached to:{" "}{selectedRule.assignments.map(({ evaluator }) => evaluator.name).join(", ")}diff --git a/web/src/features/evals/v2/server/rules/ruleRouter.ts b/web/src/features/evals/v2/server/rules/ruleRouter.tsindex 22f1c45a63..87aeb297ff 100644--- a/web/src/features/evals/v2/server/rules/ruleRouter.ts+++ b/web/src/features/evals/v2/server/rules/ruleRouter.ts@@ -238,6 +238,7 @@ export const ruleRouter = createTRPCRouter({evaluatorId: input.evaluatorId,variableMapping: input.variableMapping,},+ enableRule: input.enableRule,});}),diff --git a/web/src/features/evals/v2/server/rules/ruleService.ts b/web/src/features/evals/v2/server/rules/ruleService.tsindex e125ab57b4..a640599f10 100644--- a/web/src/features/evals/v2/server/rules/ruleService.ts+++ b/web/src/features/evals/v2/server/rules/ruleService.ts@@ -513,6 +513,7 @@ export class RuleService {projectId: string;ruleId: string;assignment: RuleAssignmentInput;+ enableRule?: boolean;}) {await this.prisma.$transaction(async (prisma) => {const rule = await this.requireRule(@@ -528,9 +529,24 @@ export class RuleService {});await repository.attachEvaluator({prisma,- ...params,+ projectId: params.projectId,+ ruleId: params.ruleId,assignment: assignment!,});+ if (params.enableRule) {+ this.assertLegacyRuleCanBeEnabled(rule.targetObject, true);+ await assertActiveRuleLimitNotExceeded({+ prisma,+ projectId: params.projectId,+ additionalActiveRules: rule.status === JobConfigState.ACTIVE ? 0 : 1,+ });+ await repository.setRuleStatus({+ prisma,+ projectId: params.projectId,+ ruleIds: [params.ruleId],+ enabled: true,+ });+ }});await invalidateProjectEvalConfigCaches(params.projectId);const rule = await this.get(params.projectId, params.ruleId);diff --git a/web/src/features/evals/v2/server/rules/ruleTypes.ts b/web/src/features/evals/v2/server/rules/ruleTypes.tsindex 4971a581c3..ab9d81675e 100644--- a/web/src/features/evals/v2/server/rules/ruleTypes.ts+++ b/web/src/features/evals/v2/server/rules/ruleTypes.ts@@ -87,6 +87,7 @@ export const SetRuleEnabledSchema = RuleIdSchema.extend({export const RuleAssignmentSchema = RuleIdSchema.extend({evaluatorId: z.string(),variableMapping: observationVariableMappingList.nullable(),+ enableRule: z.boolean().optional(),});export const RuleAssignmentIdSchema = RuleIdSchema.extend({