Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(api): keep migrated evaluation rules manageable (#16449)

Tobias Wochinger3c3ca18eed76
diff --git a/web/src/__tests__/server/unit/unstable-evals-service.servertest.ts b/web/src/__tests__/server/unit/unstable-evals-service.servertest.ts
index 6fd1ad7d72..4e0dd8c518 100644
--- a/web/src/__tests__/server/unit/unstable-evals-service.servertest.ts
+++ b/web/src/__tests__/server/unit/unstable-evals-service.servertest.ts
@@ -84,7 +84,10 @@ vi.mock("@langfuse/shared/src/db", async () => {
findFirst: vi.fn(),
create: vi.fn(),
delete: vi.fn(),
+ deleteMany: vi.fn(),
+ update: vi.fn(),
},
+ jobExecution: { deleteMany: vi.fn() },
$transaction: vi.fn(),
evaluationRuleEvaluatorAssignment: { update: vi.fn(), create: vi.fn() },
};
@@ -144,10 +147,16 @@ import {
CodeEvalJobConfigError,
assertCodeEvalRuleCanRun,
} from "@/src/features/evals/server/codeEvalJobConfigValidation";
-import { createPublicEvaluationRule } from "@/src/features/evals/server/unstable-public-api/evaluation-rule-service";
+import {
+ createPublicEvaluationRule,
+ deletePublicEvaluationRule,
+ listPublicEvaluationRules,
+ updatePublicEvaluationRule,
+} from "@/src/features/evals/server/unstable-public-api/evaluation-rule-service";
import {
findPublicV2EvaluationRule,
findPublicV2EvaluatorInFamilyOrThrow,
+ listPublicEvaluationRulePage,
} from "@/src/features/evals/server/unstable-public-api/queries";
import { MAX_ACTIVE_EVALUATION_RULES } from "@/src/features/evals/v2/server/rules/ruleErrors";
@@ -190,6 +199,25 @@ function buildCreatedRule(data: {
};
}
+function buildMigratedRuleWithLegacyFilter() {
+ const migratedRule = buildCreatedRule({
+ projectId: "project",
+ name: "Migrated rule",
+ status: "ACTIVE",
+ targetObject: "event",
+ filter: [
+ {
+ type: "stringOptions",
+ column: "experimentDatasetId",
+ operator: "any of",
+ value: ["legacy-value"],
+ },
+ ],
+ sampling: 1,
+ });
+ return migratedRule;
+}
+
describe("unstable public evaluation-rule service", () => {
beforeEach(() => {
vi.clearAllMocks();
@@ -394,4 +422,131 @@ describe("unstable public evaluation-rule service", () => {
}),
).rejects.toMatchObject({ httpCode: 400, code: "invalid_request" });
});
+
+ it("lists migrated rules with filters that do not match their current target", async () => {
+ const migratedRule = buildMigratedRuleWithLegacyFilter();
+ (listPublicEvaluationRulePage as Mock).mockResolvedValue({
+ records: [migratedRule],
+ totalItems: 1,
+ });
+
+ await expect(
+ listPublicEvaluationRules({ projectId: "project", page: 1, limit: 50 }),
+ ).resolves.toMatchObject({
+ data: [
+ {
+ id: "rule",
+ filter: [
+ {
+ type: "stringOptions",
+ column: "datasetId",
+ operator: "any of",
+ value: ["legacy-value"],
+ },
+ ],
+ },
+ ],
+ meta: { totalItems: 1 },
+ });
+ });
+
+ it("preserves migrated filters on patches that do not replace them", async () => {
+ const migratedRule = buildMigratedRuleWithLegacyFilter();
+ (findPublicV2EvaluationRule as Mock).mockResolvedValue(migratedRule);
+ (prisma.evaluationRule.findFirst as Mock).mockResolvedValue(migratedRule);
+ (prisma.evaluationRule.update as Mock).mockImplementation(({ data }) => {
+ migratedRule.name = data.name ?? migratedRule.name;
+ return migratedRule;
+ });
+
+ await expect(
+ updatePublicEvaluationRule({
+ orgId: "org",
+ projectId: "project",
+ evaluationRuleId: "rule",
+ input: { name: "Renamed rule" },
+ }),
+ ).resolves.toMatchObject({
+ name: "Renamed rule",
+ filter: [{ column: "datasetId" }],
+ });
+ expect(prisma.evaluationRule.update).toHaveBeenCalledOnce();
+ });
+
+ it("repairs migrated rules with a V2-compatible replacement filter", async () => {
+ const migratedRule = buildMigratedRuleWithLegacyFilter();
+ const replacementFilter = [
+ {
+ type: "boolean" as const,
+ column: "isRootObservation",
+ operator: "=" as const,
+ value: true,
+ },
+ ];
+ (findPublicV2EvaluationRule as Mock).mockResolvedValue(migratedRule);
+ (prisma.evaluationRule.findFirst as Mock).mockResolvedValue(migratedRule);
+ (prisma.evaluationRule.update as Mock).mockImplementation(({ data }) => {
+ migratedRule.filter = data.filter;
+ migratedRule.status = data.status;
+ return migratedRule;
+ });
+
+ await expect(
+ updatePublicEvaluationRule({
+ orgId: "org",
+ projectId: "project",
+ evaluationRuleId: "rule",
+ input: {
+ target: "observation",
+ filter: replacementFilter,
+ enabled: false,
+ },
+ }),
+ ).resolves.toMatchObject({ filter: replacementFilter, enabled: false });
+ expect(prisma.evaluationRule.update).toHaveBeenCalledOnce();
+ });
+
+ it.each([
+ {
+ operation: "disabled",
+ mutate: () =>
+ updatePublicEvaluationRule({
+ orgId: "org",
+ projectId: "project",
+ evaluationRuleId: "rule",
+ input: { enabled: false },
+ }),
+ },
+ {
+ operation: "deleted",
+ mutate: () =>
+ deletePublicEvaluationRule({
+ projectId: "project",
+ evaluationRuleId: "rule",
+ }),
+ },
+ ])(
+ "allows a migrated rule with an unsupported stored filter to be $operation",
+ async ({ mutate, operation }) => {
+ const migratedRule = buildMigratedRuleWithLegacyFilter();
+ (findPublicV2EvaluationRule as Mock).mockResolvedValue(migratedRule);
+ (prisma.evaluationRule.findFirst as Mock).mockResolvedValue(migratedRule);
+ (prisma.evaluationRule.update as Mock).mockImplementation(({ data }) => {
+ migratedRule.status = data.status ?? migratedRule.status;
+ return migratedRule;
+ });
+ (prisma.evaluationRule.deleteMany as Mock).mockResolvedValue({
+ count: 1,
+ });
+ (prisma.jobExecution.deleteMany as Mock).mockResolvedValue({ count: 0 });
+
+ const result = await mutate();
+ if (operation === "disabled") {
+ expect(result).toMatchObject({ enabled: false, status: "inactive" });
+ expect(prisma.evaluationRule.update).toHaveBeenCalledOnce();
+ } else {
+ expect(prisma.evaluationRule.deleteMany).toHaveBeenCalledOnce();
+ }
+ },
+ );
});
diff --git a/web/src/features/evals/server/unstable-public-api/adapters.ts b/web/src/features/evals/server/unstable-public-api/adapters.ts
index 05b5077de0..82a4f453d9 100644
--- a/web/src/features/evals/server/unstable-public-api/adapters.ts
+++ b/web/src/features/evals/server/unstable-public-api/adapters.ts
@@ -24,11 +24,10 @@ import { EvalTemplateType } from "@langfuse/shared/src/db";
import { logger } from "@langfuse/shared/src/server";
import { z } from "zod";
import {
- ExperimentEvaluationRuleFilter,
LegacyEvaluationRuleMapping,
- ObservationEvaluationRuleFilter,
PUBLIC_EVALUATOR_TYPE_CODE,
PUBLIC_EVALUATOR_TYPE_LLM_AS_JUDGE,
+ PublicEvaluationRuleFilter,
type PublicEvaluationRuleFilterType,
type PublicEvaluationRuleMappingType,
type PublicEvaluationRuleReadMappingType,
@@ -113,14 +112,6 @@ const INTERNAL_MAPPING_COLUMN_TO_PUBLIC_SOURCE: Record<
experiment_item_metadata: "experiment_item_metadata",
};
-function getPublicFilterArraySchema(target: PublicEvaluationRuleTargetType) {
- return z.array(
- target === "observation"
- ? ObservationEvaluationRuleFilter
- : ExperimentEvaluationRuleFilter,
- );
-}
-
export function deriveEvaluatorVariables(
template: Pick<StoredPublicEvaluatorTemplate, "vars" | "prompt">,
) {
@@ -293,18 +284,15 @@ function toStoredFilter(
return filter;
}
-function toApiFilter(
- filter: z.infer<typeof singleFilter>,
- target: PublicEvaluationRuleTargetType,
-): PublicEvaluationRuleFilterType {
- if (target === "experiment" && filter.column === "experimentDatasetId") {
+function toApiFilter(filter: z.infer<typeof singleFilter>) {
+ if (filter.column === "experimentDatasetId") {
return {
...filter,
column: "datasetId",
- } as PublicEvaluationRuleFilterType;
+ };
}
- return filter as PublicEvaluationRuleFilterType;
+ return filter;
}
export function toApiReadMappings(
@@ -404,7 +392,7 @@ function getLegacyCodeEvalVariableMapping(target: "trace" | "dataset") {
function toApiFilters(
filters: unknown,
target: PublicEvaluationRuleTargetType,
-): PublicEvaluationRuleFilterType[] {
+) {
const storedFilters = z.array(singleFilter).safeParse(filters);
if (!storedFilters.success) {
@@ -418,11 +406,10 @@ function toApiFilters(
target === "experiment"
? stripExperimentRootFilter(storedFilters.data)
: storedFilters.data;
- const publicFilters = effectiveFilters.map((filter) =>
- toApiFilter(filter, target),
- );
- const parsedPublicFilters =
- getPublicFilterArraySchema(target).safeParse(publicFilters);
+ const publicFilters = effectiveFilters.map(toApiFilter);
+ const parsedPublicFilters = z
+ .array(PublicEvaluationRuleFilter)
+ .safeParse(publicFilters);
if (!parsedPublicFilters.success) {
logger.error("Failed to parse unstable public evaluation rule filters", {
diff --git a/web/src/features/evals/server/unstable-public-api/evaluation-rule-service.ts b/web/src/features/evals/server/unstable-public-api/evaluation-rule-service.ts
index ef7c36bacc..40cef77bc6 100644
--- a/web/src/features/evals/server/unstable-public-api/evaluation-rule-service.ts
+++ b/web/src/features/evals/server/unstable-public-api/evaluation-rule-service.ts
@@ -569,28 +569,29 @@ export async function updatePublicEvaluationRule(params: {
if (nextEnabled && existingPublic.status !== "active") {
await assertActivePublicApiEvaluationRuleLimitNotExceeded(params.projectId);
}
- const nextTarget =
- "target" in params.input && params.input.target !== undefined
- ? params.input.target
- : existingPublic.target;
- if ("filter" in params.input && params.input.filter !== undefined) {
+ const suppliedTarget =
+ "target" in params.input ? params.input.target : undefined;
+ const suppliedFilter =
+ "filter" in params.input ? params.input.filter : undefined;
+ const nextTarget = suppliedTarget ?? existingPublic.target;
+ if (suppliedFilter !== undefined) {
await assertEvaluationRuleFilterValuesExistForProject({
projectId: params.projectId,
target: nextTarget,
- filters: params.input.filter,
+ filters: suppliedFilter,
});
}
- const nextFilter =
- "filter" in params.input && params.input.filter !== undefined
- ? params.input.filter
- : existingPublic.filter;
-
+ const updatesFilterOrTarget =
+ suppliedFilter !== undefined || suppliedTarget !== undefined;
const ruleFields = toEvaluationRuleFields({
name: params.input.name ?? existingPublic.name,
target: nextTarget,
enabled: nextEnabled,
sampling: params.input.sampling ?? existingPublic.sampling,
- filter: nextFilter,
+ // Target changes must also validate the effective filter against the new target.
+ filter: updatesFilterOrTarget
+ ? (suppliedFilter ?? existingPublic.filter)
+ : [],
});
const firstAssignment = existing.assignments[0];
@@ -687,9 +688,11 @@ export async function updatePublicEvaluationRule(params: {
orgId: params.orgId,
projectId: params.projectId,
assignments: effectiveAssignments,
- targetObject: ruleFields.targetObject as EvalTargetObject,
+ targetObject: ruleFields.targetObject,
scoreName: ruleFields.scoreName,
- filter: ruleFields.filter,
+ filter: updatesFilterOrTarget
+ ? ruleFields.filter
+ : (existing.filter as FilterCondition[]),
});
}
@@ -774,14 +777,20 @@ async function updateRuleWithRuleService(params: {
ruleService.update({
projectId: params.projectId,
ruleId: params.evaluationRuleId,
- targetObject: params.fields.targetObject as EvalTargetObject,
- name: params.fields.scoreName,
+ targetObject:
+ "target" in params.input ? params.fields.targetObject : undefined,
+ name: params.input.name,
enabled:
- params.effectiveAssignmentCount > 0 &&
- params.fields.status === JobConfigState.ACTIVE,
- sampling: params.fields.sampling,
- filter: params.fields.filter as FilterState,
- ...(evaluatorMappings === undefined ? {} : { evaluatorMappings }),
+ params.input.enabled === undefined
+ ? undefined
+ : params.effectiveAssignmentCount > 0 &&
+ params.fields.status === JobConfigState.ACTIVE,
+ sampling: params.input.sampling,
+ filter:
+ "filter" in params.input || "target" in params.input
+ ? (params.fields.filter as FilterState)
+ : undefined,
+ evaluatorMappings,
}),
);
return findPublicV2EvaluationRuleOrThrow({