Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(evals): activate disabled rules on evaluator attach (#16465)

Tobias Wochinger81b1e751ae19
diff --git a/web/src/__tests__/server/evaluation-rule-v2-service.servertest.ts b/web/src/__tests__/server/evaluation-rule-v2-service.servertest.ts
index 255b128946..0644077bb5 100644
--- a/web/src/__tests__/server/evaluation-rule-v2-service.servertest.ts
+++ b/web/src/__tests__/server/evaluation-rule-v2-service.servertest.ts
@@ -717,6 +717,26 @@ describe("RuleService", () => {
});
});
+ it("enables a disabled rule when attaching with activation", async () => {
+ const evaluator = await createEvaluator();
+ const service = createService();
+ const rule = await service.create(
+ { ...createInput(), enabled: false },
+ null,
+ );
+ const attachment = {
+ projectId,
+ ruleId: rule.id,
+ assignment: { evaluatorId: evaluator.id, variableMapping: null },
+ enableRule: true,
+ };
+
+ await expect(service.attach(attachment)).resolves.toMatchObject({
+ enabled: true,
+ assignments: [expect.objectContaining({ evaluatorId: evaluator.id })],
+ });
+ });
+
it("updates experiment filters and assignments", async () => {
const [first, second] = await Promise.all([
createEvaluator(),
diff --git a/web/src/__tests__/server/evaluation-rule-v2-trpc.servertest.ts b/web/src/__tests__/server/evaluation-rule-v2-trpc.servertest.ts
index cc828a40fa..b2f6ac1c90 100644
--- a/web/src/__tests__/server/evaluation-rule-v2-trpc.servertest.ts
+++ b/web/src/__tests__/server/evaluation-rule-v2-trpc.servertest.ts
@@ -252,12 +252,15 @@ describe("evaluation rule v2 tRPC", () => {
).resolves.toBe(0);
});
- it("supports attaching and detaching an evaluator", async () => {
+ it("supports attaching with activation and detaching an evaluator", async () => {
const [first, second] = await Promise.all([
createEvaluator("First transport evaluator"),
createEvaluator("Second transport evaluator"),
]);
- const rule = await caller.evalsV2.rules.create(ruleInput(first.id));
+ const rule = await caller.evalsV2.rules.create({
+ ...ruleInput(first.id),
+ enabled: false,
+ });
const attached = await caller.evalsV2.rules.attach({
projectId,
ruleId: rule.id,
@@ -265,8 +268,10 @@ describe("evaluation rule v2 tRPC", () => {
variableMapping: [
{ templateVariable: "output", selectedColumnId: "input" },
],
+ enableRule: true,
});
+ expect(attached.enabled).toBe(true);
expect(attached.assignments).toEqual(
expect.arrayContaining([
expect.objectContaining({ evaluatorId: second.id }),
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsx
index 1935c06a23..33eb0a3efb 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSavedDialogContainer/EvaluatorSavedDialogContainer.tsx
@@ -69,7 +69,7 @@ export function EvaluatorSavedDialogContainer({
evaluator.hasCompletedTestCall ?? false,
);
const missingCostTestRequest = useRef<Promise<void> | null>(null);
- const rules = api.evalsV2.rules.list.useQuery(
+ const activeRules = api.evalsV2.rules.list.useQuery(
{
projectId,
page: 1,
@@ -79,13 +79,27 @@ export function EvaluatorSavedDialogContainer({
},
{ enabled: dialogPhase === "saved" },
);
+ const inactiveRules = api.evalsV2.rules.list.useQuery(
+ {
+ projectId,
+ page: 1,
+ limit: 100,
+ enabled: false,
+ targetObjects: [EvalTargetObject.EVENT, EvalTargetObject.EXPERIMENT],
+ },
+ { enabled: dialogPhase === "saved" },
+ );
const availableRules = useMemo(
() =>
- [...(rules.data?.rules ?? [])].sort(
+ [
+ ...(activeRules.data?.rules ?? []),
+ ...(inactiveRules.data?.rules ?? []),
+ ].sort(
(left, right) => right.assignments.length - left.assignments.length,
),
- [rules.data?.rules],
+ [activeRules.data?.rules, inactiveRules.data?.rules],
);
+ const rulesPending = activeRules.isPending || inactiveRules.isPending;
const {
supportedFilters: supportedRuleFilters,
unsupportedReasons: unsupportedRuleFilterReasons,
@@ -126,6 +140,7 @@ export function EvaluatorSavedDialogContainer({
ruleId: rule.id,
evaluatorId: evaluator.id,
variableMapping: null,
+ enableRule: !rule.enabled,
});
capture("evaluation_rules:attach_evaluator", {
evaluatorCount: 1,
@@ -285,7 +300,7 @@ export function EvaluatorSavedDialogContainer({
if (
mode !== "different-scope" ||
selectedRuleId !== undefined ||
- rules.isPending
+ rulesPending
) {
return;
}
@@ -296,13 +311,7 @@ export function EvaluatorSavedDialogContainer({
} else {
setSelectedRuleId(null);
}
- }, [
- availableRules,
- mode,
- rules.isPending,
- selectExistingRule,
- selectedRuleId,
- ]);
+ }, [availableRules, mode, rulesPending, selectExistingRule, selectedRuleId]);
const handleModeChange = (nextMode: EvaluatorSavedMode) => {
estimateRequestId.current += 1;
@@ -314,7 +323,7 @@ export function EvaluatorSavedDialogContainer({
return;
}
setSelectedRuleId(
- nextMode === "different-scope" && !rules.isPending ? null : undefined,
+ nextMode === "different-scope" && !rulesPending ? null : undefined,
);
setIsEstimating(false);
if (nextMode === "test-filters") {
@@ -349,7 +358,7 @@ export function EvaluatorSavedDialogContainer({
onOpenChange={setRulePickerOpen}
disabledRules={[]}
availableRules={availableRules}
- loading={rules.isPending}
+ loading={rulesPending}
onSelectAvailableRule={selectExistingRule}
onCreateRule={selectNewRule}
>
@@ -395,7 +404,7 @@ export function EvaluatorSavedDialogContainer({
/>
{selectedRule.assignments.length > 0 ? (
<p className="text-muted-foreground text-sm">
- Already running on this rule:{" "}
+ Already attached to:{" "}
{selectedRule.assignments
.map(({ evaluator }) => evaluator.name)
.join(", ")}
diff --git a/web/src/features/evals/v2/server/rules/ruleRouter.ts b/web/src/features/evals/v2/server/rules/ruleRouter.ts
index 22f1c45a63..87aeb297ff 100644
--- a/web/src/features/evals/v2/server/rules/ruleRouter.ts
+++ b/web/src/features/evals/v2/server/rules/ruleRouter.ts
@@ -238,6 +238,7 @@ export const ruleRouter = createTRPCRouter({
evaluatorId: input.evaluatorId,
variableMapping: input.variableMapping,
},
+ enableRule: input.enableRule,
});
}),
diff --git a/web/src/features/evals/v2/server/rules/ruleService.ts b/web/src/features/evals/v2/server/rules/ruleService.ts
index e125ab57b4..a640599f10 100644
--- a/web/src/features/evals/v2/server/rules/ruleService.ts
+++ b/web/src/features/evals/v2/server/rules/ruleService.ts
@@ -513,6 +513,7 @@ export class RuleService {
projectId: string;
ruleId: string;
assignment: RuleAssignmentInput;
+ enableRule?: boolean;
}) {
await this.prisma.$transaction(async (prisma) => {
const rule = await this.requireRule(
@@ -528,9 +529,24 @@ export class RuleService {
});
await repository.attachEvaluator({
prisma,
- ...params,
+ projectId: params.projectId,
+ ruleId: params.ruleId,
assignment: assignment!,
});
+ if (params.enableRule) {
+ this.assertLegacyRuleCanBeEnabled(rule.targetObject, true);
+ await assertActiveRuleLimitNotExceeded({
+ prisma,
+ projectId: params.projectId,
+ additionalActiveRules: rule.status === JobConfigState.ACTIVE ? 0 : 1,
+ });
+ await repository.setRuleStatus({
+ prisma,
+ projectId: params.projectId,
+ ruleIds: [params.ruleId],
+ enabled: true,
+ });
+ }
});
await invalidateProjectEvalConfigCaches(params.projectId);
const rule = await this.get(params.projectId, params.ruleId);
diff --git a/web/src/features/evals/v2/server/rules/ruleTypes.ts b/web/src/features/evals/v2/server/rules/ruleTypes.ts
index 4971a581c3..ab9d81675e 100644
--- a/web/src/features/evals/v2/server/rules/ruleTypes.ts
+++ b/web/src/features/evals/v2/server/rules/ruleTypes.ts
@@ -87,6 +87,7 @@ export const SetRuleEnabledSchema = RuleIdSchema.extend({
export const RuleAssignmentSchema = RuleIdSchema.extend({
evaluatorId: z.string(),
variableMapping: observationVariableMappingList.nullable(),
+ enableRule: z.boolean().optional(),
});
export const RuleAssignmentIdSchema = RuleIdSchema.extend({