🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(api): keep migrated evaluation rules manageable (#16449)
Tobias Wochinger3c3ca18eed76
diff --git a/web/src/__tests__/server/unit/unstable-evals-service.servertest.ts b/web/src/__tests__/server/unit/unstable-evals-service.servertest.tsindex 6fd1ad7d72..4e0dd8c518 100644--- a/web/src/__tests__/server/unit/unstable-evals-service.servertest.ts+++ b/web/src/__tests__/server/unit/unstable-evals-service.servertest.ts@@ -84,7 +84,10 @@ vi.mock("@langfuse/shared/src/db", async () => {findFirst: vi.fn(),create: vi.fn(),delete: vi.fn(),+ deleteMany: vi.fn(),+ update: vi.fn(),},+ jobExecution: { deleteMany: vi.fn() },$transaction: vi.fn(),evaluationRuleEvaluatorAssignment: { update: vi.fn(), create: vi.fn() },};@@ -144,10 +147,16 @@ import {CodeEvalJobConfigError,assertCodeEvalRuleCanRun,} from "@/src/features/evals/server/codeEvalJobConfigValidation";-import { createPublicEvaluationRule } from "@/src/features/evals/server/unstable-public-api/evaluation-rule-service";+import {+ createPublicEvaluationRule,+ deletePublicEvaluationRule,+ listPublicEvaluationRules,+ updatePublicEvaluationRule,+} from "@/src/features/evals/server/unstable-public-api/evaluation-rule-service";import {findPublicV2EvaluationRule,findPublicV2EvaluatorInFamilyOrThrow,+ listPublicEvaluationRulePage,} from "@/src/features/evals/server/unstable-public-api/queries";import { MAX_ACTIVE_EVALUATION_RULES } from "@/src/features/evals/v2/server/rules/ruleErrors";@@ -190,6 +199,25 @@ function buildCreatedRule(data: {};}+function buildMigratedRuleWithLegacyFilter() {+ const migratedRule = buildCreatedRule({+ projectId: "project",+ name: "Migrated rule",+ status: "ACTIVE",+ targetObject: "event",+ filter: [+ {+ type: "stringOptions",+ column: "experimentDatasetId",+ operator: "any of",+ value: ["legacy-value"],+ },+ ],+ sampling: 1,+ });+ return migratedRule;+}+describe("unstable public evaluation-rule service", () => {beforeEach(() => {vi.clearAllMocks();@@ -394,4 +422,131 @@ describe("unstable public evaluation-rule service", () => {}),).rejects.toMatchObject({ httpCode: 400, code: "invalid_request" });});++ it("lists migrated rules with filters that do not match their current target", async () => {+ const migratedRule = buildMigratedRuleWithLegacyFilter();+ (listPublicEvaluationRulePage as Mock).mockResolvedValue({+ records: [migratedRule],+ totalItems: 1,+ });++ await expect(+ listPublicEvaluationRules({ projectId: "project", page: 1, limit: 50 }),+ ).resolves.toMatchObject({+ data: [+ {+ id: "rule",+ filter: [+ {+ type: "stringOptions",+ column: "datasetId",+ operator: "any of",+ value: ["legacy-value"],+ },+ ],+ },+ ],+ meta: { totalItems: 1 },+ });+ });++ it("preserves migrated filters on patches that do not replace them", async () => {+ const migratedRule = buildMigratedRuleWithLegacyFilter();+ (findPublicV2EvaluationRule as Mock).mockResolvedValue(migratedRule);+ (prisma.evaluationRule.findFirst as Mock).mockResolvedValue(migratedRule);+ (prisma.evaluationRule.update as Mock).mockImplementation(({ data }) => {+ migratedRule.name = data.name ?? migratedRule.name;+ return migratedRule;+ });++ await expect(+ updatePublicEvaluationRule({+ orgId: "org",+ projectId: "project",+ evaluationRuleId: "rule",+ input: { name: "Renamed rule" },+ }),+ ).resolves.toMatchObject({+ name: "Renamed rule",+ filter: [{ column: "datasetId" }],+ });+ expect(prisma.evaluationRule.update).toHaveBeenCalledOnce();+ });++ it("repairs migrated rules with a V2-compatible replacement filter", async () => {+ const migratedRule = buildMigratedRuleWithLegacyFilter();+ const replacementFilter = [+ {+ type: "boolean" as const,+ column: "isRootObservation",+ operator: "=" as const,+ value: true,+ },+ ];+ (findPublicV2EvaluationRule as Mock).mockResolvedValue(migratedRule);+ (prisma.evaluationRule.findFirst as Mock).mockResolvedValue(migratedRule);+ (prisma.evaluationRule.update as Mock).mockImplementation(({ data }) => {+ migratedRule.filter = data.filter;+ migratedRule.status = data.status;+ return migratedRule;+ });++ await expect(+ updatePublicEvaluationRule({+ orgId: "org",+ projectId: "project",+ evaluationRuleId: "rule",+ input: {+ target: "observation",+ filter: replacementFilter,+ enabled: false,+ },+ }),+ ).resolves.toMatchObject({ filter: replacementFilter, enabled: false });+ expect(prisma.evaluationRule.update).toHaveBeenCalledOnce();+ });++ it.each([+ {+ operation: "disabled",+ mutate: () =>+ updatePublicEvaluationRule({+ orgId: "org",+ projectId: "project",+ evaluationRuleId: "rule",+ input: { enabled: false },+ }),+ },+ {+ operation: "deleted",+ mutate: () =>+ deletePublicEvaluationRule({+ projectId: "project",+ evaluationRuleId: "rule",+ }),+ },+ ])(+ "allows a migrated rule with an unsupported stored filter to be $operation",+ async ({ mutate, operation }) => {+ const migratedRule = buildMigratedRuleWithLegacyFilter();+ (findPublicV2EvaluationRule as Mock).mockResolvedValue(migratedRule);+ (prisma.evaluationRule.findFirst as Mock).mockResolvedValue(migratedRule);+ (prisma.evaluationRule.update as Mock).mockImplementation(({ data }) => {+ migratedRule.status = data.status ?? migratedRule.status;+ return migratedRule;+ });+ (prisma.evaluationRule.deleteMany as Mock).mockResolvedValue({+ count: 1,+ });+ (prisma.jobExecution.deleteMany as Mock).mockResolvedValue({ count: 0 });++ const result = await mutate();+ if (operation === "disabled") {+ expect(result).toMatchObject({ enabled: false, status: "inactive" });+ expect(prisma.evaluationRule.update).toHaveBeenCalledOnce();+ } else {+ expect(prisma.evaluationRule.deleteMany).toHaveBeenCalledOnce();+ }+ },+ );});diff --git a/web/src/features/evals/server/unstable-public-api/adapters.ts b/web/src/features/evals/server/unstable-public-api/adapters.tsindex 05b5077de0..82a4f453d9 100644--- a/web/src/features/evals/server/unstable-public-api/adapters.ts+++ b/web/src/features/evals/server/unstable-public-api/adapters.ts@@ -24,11 +24,10 @@ import { EvalTemplateType } from "@langfuse/shared/src/db";import { logger } from "@langfuse/shared/src/server";import { z } from "zod";import {- ExperimentEvaluationRuleFilter,LegacyEvaluationRuleMapping,- ObservationEvaluationRuleFilter,PUBLIC_EVALUATOR_TYPE_CODE,PUBLIC_EVALUATOR_TYPE_LLM_AS_JUDGE,+ PublicEvaluationRuleFilter,type PublicEvaluationRuleFilterType,type PublicEvaluationRuleMappingType,type PublicEvaluationRuleReadMappingType,@@ -113,14 +112,6 @@ const INTERNAL_MAPPING_COLUMN_TO_PUBLIC_SOURCE: Record<experiment_item_metadata: "experiment_item_metadata",};-function getPublicFilterArraySchema(target: PublicEvaluationRuleTargetType) {- return z.array(- target === "observation"- ? ObservationEvaluationRuleFilter- : ExperimentEvaluationRuleFilter,- );-}-export function deriveEvaluatorVariables(template: Pick<StoredPublicEvaluatorTemplate, "vars" | "prompt">,) {@@ -293,18 +284,15 @@ function toStoredFilter(return filter;}-function toApiFilter(- filter: z.infer<typeof singleFilter>,- target: PublicEvaluationRuleTargetType,-): PublicEvaluationRuleFilterType {- if (target === "experiment" && filter.column === "experimentDatasetId") {+function toApiFilter(filter: z.infer<typeof singleFilter>) {+ if (filter.column === "experimentDatasetId") {return {...filter,column: "datasetId",- } as PublicEvaluationRuleFilterType;+ };}- return filter as PublicEvaluationRuleFilterType;+ return filter;}export function toApiReadMappings(@@ -404,7 +392,7 @@ function getLegacyCodeEvalVariableMapping(target: "trace" | "dataset") {function toApiFilters(filters: unknown,target: PublicEvaluationRuleTargetType,-): PublicEvaluationRuleFilterType[] {+) {const storedFilters = z.array(singleFilter).safeParse(filters);if (!storedFilters.success) {@@ -418,11 +406,10 @@ function toApiFilters(target === "experiment"? stripExperimentRootFilter(storedFilters.data): storedFilters.data;- const publicFilters = effectiveFilters.map((filter) =>- toApiFilter(filter, target),- );- const parsedPublicFilters =- getPublicFilterArraySchema(target).safeParse(publicFilters);+ const publicFilters = effectiveFilters.map(toApiFilter);+ const parsedPublicFilters = z+ .array(PublicEvaluationRuleFilter)+ .safeParse(publicFilters);if (!parsedPublicFilters.success) {logger.error("Failed to parse unstable public evaluation rule filters", {diff --git a/web/src/features/evals/server/unstable-public-api/evaluation-rule-service.ts b/web/src/features/evals/server/unstable-public-api/evaluation-rule-service.tsindex ef7c36bacc..40cef77bc6 100644--- a/web/src/features/evals/server/unstable-public-api/evaluation-rule-service.ts+++ b/web/src/features/evals/server/unstable-public-api/evaluation-rule-service.ts@@ -569,28 +569,29 @@ export async function updatePublicEvaluationRule(params: {if (nextEnabled && existingPublic.status !== "active") {await assertActivePublicApiEvaluationRuleLimitNotExceeded(params.projectId);}- const nextTarget =- "target" in params.input && params.input.target !== undefined- ? params.input.target- : existingPublic.target;- if ("filter" in params.input && params.input.filter !== undefined) {+ const suppliedTarget =+ "target" in params.input ? params.input.target : undefined;+ const suppliedFilter =+ "filter" in params.input ? params.input.filter : undefined;+ const nextTarget = suppliedTarget ?? existingPublic.target;+ if (suppliedFilter !== undefined) {await assertEvaluationRuleFilterValuesExistForProject({projectId: params.projectId,target: nextTarget,- filters: params.input.filter,+ filters: suppliedFilter,});}- const nextFilter =- "filter" in params.input && params.input.filter !== undefined- ? params.input.filter- : existingPublic.filter;-+ const updatesFilterOrTarget =+ suppliedFilter !== undefined || suppliedTarget !== undefined;const ruleFields = toEvaluationRuleFields({name: params.input.name ?? existingPublic.name,target: nextTarget,enabled: nextEnabled,sampling: params.input.sampling ?? existingPublic.sampling,- filter: nextFilter,+ // Target changes must also validate the effective filter against the new target.+ filter: updatesFilterOrTarget+ ? (suppliedFilter ?? existingPublic.filter)+ : [],});const firstAssignment = existing.assignments[0];@@ -687,9 +688,11 @@ export async function updatePublicEvaluationRule(params: {orgId: params.orgId,projectId: params.projectId,assignments: effectiveAssignments,- targetObject: ruleFields.targetObject as EvalTargetObject,+ targetObject: ruleFields.targetObject,scoreName: ruleFields.scoreName,- filter: ruleFields.filter,+ filter: updatesFilterOrTarget+ ? ruleFields.filter+ : (existing.filter as FilterCondition[]),});}@@ -774,14 +777,20 @@ async function updateRuleWithRuleService(params: {ruleService.update({projectId: params.projectId,ruleId: params.evaluationRuleId,- targetObject: params.fields.targetObject as EvalTargetObject,- name: params.fields.scoreName,+ targetObject:+ "target" in params.input ? params.fields.targetObject : undefined,+ name: params.input.name,enabled:- params.effectiveAssignmentCount > 0 &&- params.fields.status === JobConfigState.ACTIVE,- sampling: params.fields.sampling,- filter: params.fields.filter as FilterState,- ...(evaluatorMappings === undefined ? {} : { evaluatorMappings }),+ params.input.enabled === undefined+ ? undefined+ : params.effectiveAssignmentCount > 0 &&+ params.fields.status === JobConfigState.ACTIVE,+ sampling: params.input.sampling,+ filter:+ "filter" in params.input || "target" in params.input+ ? (params.fields.filter as FilterState)+ : undefined,+ evaluatorMappings,}),);return findPublicV2EvaluationRuleOrThrow({