🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(api): keep historical evaluation rule filters readable (#16819)
Tobias Wochinger4e0ce09bbef6
diff --git a/fern/apis/server/definition/unstable/evaluation-rules.yml b/fern/apis/server/definition/unstable/evaluation-rules.ymlindex 26240d0f13..a347e1e506 100644--- a/fern/apis/server/definition/unstable/evaluation-rules.yml+++ b/fern/apis/server/definition/unstable/evaluation-rules.yml@@ -2,6 +2,7 @@imports:commons: ./commons.ymlerrors: ./errors.yml+ evals: ../evaluation-commons.ymlpagination: ../utils/pagination.ymlservice:@@ -323,7 +324,7 @@ service:type: stringdocs: Evaluation rule identifier.request: UpdateEvaluationRuleRequest- response: EvaluationRule+ response: ReadableV2EvaluationRuleerrors:- errors.BadRequestError- errors.UnauthorizedError@@ -431,6 +432,22 @@ types:mapping:type: list<commons.PromptVariableMappingRead>docs: Deprecated compatibility alias containing the effective mapping for `evaluators[0]`.++ ReadableV2EvaluationRule:+ extends: EvaluationRuleBase+ properties:+ evaluators:+ type: list<EvaluationRuleEvaluatorAssignment>+ docs: Evaluators attached to this rule in deterministic assignment order. A `null` mapping inherits the evaluator version's default mapping.+ target:+ type: commons.EvaluationRuleTarget+ docs: Target object type that should trigger scoring.+ filter:+ type: list<evals.EvaluationRuleReadFilter>+ docs: List of stored filter conditions returned verbatim. Historical filters remain readable even when the current write contract no longer accepts their shape.+ mapping:+ type: list<commons.PromptVariableMappingRead>+ docs: Deprecated compatibility alias containing the effective mapping for `evaluators[0]`.examples:- value:id: erule_123@@ -498,7 +515,7 @@ types:docs: Evaluation rule returned by list and get, including read-only legacy trace and dataset rules.discriminated: falseunion:- - EvaluationRule+ - ReadableV2EvaluationRule- LegacyEvaluationRuleEvaluationRules:diff --git a/web/public/generated/api/openapi.yml b/web/public/generated/api/openapi.ymlindex e6541a1766..f5e6233254 100644--- a/web/public/generated/api/openapi.yml+++ b/web/public/generated/api/openapi.yml@@ -8973,7 +8973,7 @@ paths:content:application/json:schema:- $ref: '#/components/schemas/unstableEvaluationRule'+ $ref: '#/components/schemas/unstableReadableV2EvaluationRule''400':description: ''content:@@ -19090,6 +19090,42 @@ components:- mappingallOf:- $ref: '#/components/schemas/unstableEvaluationRuleBase'+ unstableReadableV2EvaluationRule:+ title: unstableReadableV2EvaluationRule+ type: object+ properties:+ evaluators:+ type: array+ items:+ $ref: '#/components/schemas/unstableEvaluationRuleEvaluatorAssignment'+ description: >-+ Evaluators attached to this rule in deterministic assignment order.+ A `null` mapping inherits the evaluator version's default mapping.+ target:+ $ref: '#/components/schemas/unstableEvaluationRuleTarget'+ description: Target object type that should trigger scoring.+ filter:+ type: array+ items:+ $ref: '#/components/schemas/EvaluationRuleReadFilter'+ description: >-+ List of stored filter conditions returned verbatim. Historical+ filters remain readable even when the current write contract no+ longer accepts their shape.+ mapping:+ type: array+ items:+ $ref: '#/components/schemas/unstablePromptVariableMappingRead'+ description: >-+ Deprecated compatibility alias containing the effective mapping for+ `evaluators[0]`.+ required:+ - evaluators+ - target+ - filter+ - mapping+ allOf:+ - $ref: '#/components/schemas/unstableEvaluationRuleBase'unstableLegacyEvaluationRule:title: unstableLegacyEvaluationRuletype: object@@ -19144,7 +19180,7 @@ components:unstableReadableEvaluationRule:title: unstableReadableEvaluationRuleoneOf:- - $ref: '#/components/schemas/unstableEvaluationRule'+ - $ref: '#/components/schemas/unstableReadableV2EvaluationRule'- $ref: '#/components/schemas/unstableLegacyEvaluationRule'description: >-Evaluation rule returned by list and get, including read-only legacydiff --git a/web/src/__tests__/server/unit/unstable-evals-service.servertest.ts b/web/src/__tests__/server/unit/unstable-evals-service.servertest.tsindex 4e0dd8c518..f1918047a1 100644--- a/web/src/__tests__/server/unit/unstable-evals-service.servertest.ts+++ b/web/src/__tests__/server/unit/unstable-evals-service.servertest.ts@@ -207,10 +207,16 @@ function buildMigratedRuleWithLegacyFilter() {targetObject: "event",filter: [{- type: "stringOptions",- column: "experimentDatasetId",- operator: "any of",- value: ["legacy-value"],+ type: "string",+ column: "environment",+ operator: "does not contain",+ value: "langfuse-",+ },+ {+ type: "null",+ column: "experimentId",+ operator: "is null",+ value: "",},],sampling: 1,@@ -438,10 +444,16 @@ describe("unstable public evaluation-rule service", () => {id: "rule",filter: [{- type: "stringOptions",- column: "datasetId",- operator: "any of",- value: ["legacy-value"],+ type: "string",+ column: "environment",+ operator: "does not contain",+ value: "langfuse-",+ },+ {+ type: "null",+ column: "experimentId",+ operator: "is null",+ value: "",},],},@@ -464,11 +476,11 @@ describe("unstable public evaluation-rule service", () => {orgId: "org",projectId: "project",evaluationRuleId: "rule",- input: { name: "Renamed rule" },+ input: { name: "Renamed rule", target: "observation" },}),).resolves.toMatchObject({name: "Renamed rule",- filter: [{ column: "datasetId" }],+ filter: [{ column: "environment" }, { column: "experimentId" }],});expect(prisma.evaluationRule.update).toHaveBeenCalledOnce();});@@ -506,6 +518,26 @@ describe("unstable public evaluation-rule service", () => {expect(prisma.evaluationRule.update).toHaveBeenCalledOnce();});+ it("requires a replacement filter when changing the target of a migrated rule", async () => {+ const migratedRule = buildMigratedRuleWithLegacyFilter();+ (findPublicV2EvaluationRule as Mock).mockResolvedValue(migratedRule);+ (prisma.evaluationRule.findFirst as Mock).mockResolvedValue(migratedRule);++ await expect(+ updatePublicEvaluationRule({+ orgId: "org",+ projectId: "project",+ evaluationRuleId: "rule",+ input: { target: "experiment" },+ }),+ ).rejects.toMatchObject({+ httpCode: 400,+ code: "invalid_body",+ details: { field: "filter" },+ });+ expect(prisma.evaluationRule.update).not.toHaveBeenCalled();+ });+it.each([{operation: "disabled",diff --git a/web/src/features/evals/server/unstable-public-api/adapters.ts b/web/src/features/evals/server/unstable-public-api/adapters.tsindex 1cbb70820a..76aed05d5e 100644--- a/web/src/features/evals/server/unstable-public-api/adapters.ts+++ b/web/src/features/evals/server/unstable-public-api/adapters.ts@@ -28,6 +28,7 @@ import {PUBLIC_EVALUATOR_TYPE_CODE,PUBLIC_EVALUATOR_TYPE_LLM_AS_JUDGE,PublicEvaluationRuleFilter,+ PublicEvaluationRuleReadFilter,type PublicEvaluationRuleFilterType,type PromptVariableMappingInputType,type PromptVariableMappingReadType,@@ -406,19 +407,9 @@ function toApiFilters(target === "experiment"? stripExperimentRootFilter(storedFilters.data): storedFilters.data;- const publicFilters = effectiveFilters.map(toApiFilter);- const parsedPublicFilters = z- .array(PublicEvaluationRuleFilter)- .safeParse(publicFilters);-- if (!parsedPublicFilters.success) {- logger.error("Failed to parse unstable public evaluation rule filters", {- issues: parsedPublicFilters.error.issues,- });- throw new InternalServerError("Evaluation rule filter is corrupted");- }-- return parsedPublicFilters.data;+ return z+ .array(PublicEvaluationRuleReadFilter)+ .parse(effectiveFilters.map(toApiFilter));}export function toApiEvaluator(params: {@@ -614,7 +605,15 @@ export function toApiWritableV2EvaluationRule() {throw new InternalServerError("Evaluation rule target is corrupted");}- return evaluationRule;++ const filter = z+ .array(PublicEvaluationRuleFilter)+ .safeParse(evaluationRule.filter);+ if (!filter.success) {+ throw new InternalServerError("Evaluation rule filter is corrupted");+ }++ return { ...evaluationRule, filter: filter.data };}export function toEvaluationRuleInput(params: {diff --git a/web/src/features/evals/server/unstable-public-api/evaluation-rule-service.ts b/web/src/features/evals/server/unstable-public-api/evaluation-rule-service.tsindex c5161c67ba..4cdf9f752c 100644--- a/web/src/features/evals/server/unstable-public-api/evaluation-rule-service.ts+++ b/web/src/features/evals/server/unstable-public-api/evaluation-rule-service.ts@@ -2,6 +2,7 @@ import { type ApiAccessScope } from "@langfuse/shared/src/server";import { EvalTemplateType, prisma } from "@langfuse/shared/src/db";import {JobConfigState,+ InternalServerError,LangfuseNotFoundError,EvalTargetObject,type FilterCondition,@@ -23,6 +24,7 @@ import type {PostUnstableEvaluationRuleBodyType,} from "@/src/features/public-api/types/unstable-evaluation-rules";import {+ PublicEvaluationRuleFilter,type PromptVariableMappingInputType,type PromptVariableMappingReadType,type PublicEvaluationRuleTargetType,@@ -276,6 +278,17 @@ async function findPublicWritableV2EvaluationRuleOrThrow(params: {return rule;}+function toApiReadableV2EvaluationRule(rule: StoredPublicV2EvaluationRule) {+ const evaluationRule = toApiV2EvaluationRule(rule);+ if (+ evaluationRule.target !== "observation" &&+ evaluationRule.target !== "experiment"+ ) {+ throw new InternalServerError("Evaluation rule target is corrupted");+ }+ return evaluationRule;+}+export async function listPublicEvaluationRules(params: {projectId: string;page: number;@@ -564,7 +577,7 @@ export async function updatePublicEvaluationRule(params: {auditScope?: Pick<ApiAccessScope, "orgId" | "apiKeyId">;}) {const existing = await findPublicWritableV2EvaluationRuleOrThrow(params);- const existingPublic = toApiWritableV2EvaluationRule(existing);+ const existingPublic = toApiReadableV2EvaluationRule(existing);const nextEnabled = params.input.enabled ?? existingPublic.enabled;if (nextEnabled && existingPublic.status !== "active") {await assertActivePublicApiEvaluationRuleLimitNotExceeded(params.projectId);@@ -582,7 +595,24 @@ export async function updatePublicEvaluationRule(params: {});}const updatesFilterOrTarget =- suppliedFilter !== undefined || suppliedTarget !== undefined;+ suppliedFilter !== undefined ||+ (suppliedTarget !== undefined && suppliedTarget !== existingPublic.target);+ const parsedExistingFilter = PublicEvaluationRuleFilter.array().safeParse(+ existingPublic.filter,+ );+ if (+ updatesFilterOrTarget &&+ suppliedFilter === undefined &&+ !parsedExistingFilter.success+ ) {+ throw createStructuredPublicApiError({+ httpCode: 400,+ code: "invalid_body",+ message:+ "The stored filter is no longer accepted for writes. Provide a target-compatible filter with this update.",+ details: { field: "filter" },+ });+ }const ruleFields = toEvaluationRuleFields({name: params.input.name ?? existingPublic.name,target: nextTarget,@@ -590,7 +620,7 @@ export async function updatePublicEvaluationRule(params: {sampling: params.input.sampling ?? existingPublic.sampling,// Target changes must also validate the effective filter against the new target.filter: updatesFilterOrTarget- ? (suppliedFilter ?? existingPublic.filter)+ ? (suppliedFilter ?? parsedExistingFilter.data ?? []): [],});@@ -705,8 +735,9 @@ export async function updatePublicEvaluationRule(params: {replacementAssignments,patchedFirstAssignment,effectiveAssignmentCount: effectiveAssignments.length,+ updatesFilterOrTarget,});- const evaluationRule = toApiWritableV2EvaluationRule(updated);+ const evaluationRule = toApiReadableV2EvaluationRule(updated);if (params.auditScope) {await auditLog({action: "update",@@ -745,6 +776,7 @@ async function updateRuleWithRuleService(params: {replacementAssignments: PreparedAssignment[] | null;patchedFirstAssignment: PreparedAssignment | null;effectiveAssignmentCount: number;+ updatesFilterOrTarget: boolean;}) {const firstAssignment = params.existing.assignments[0];const evaluatorMappings = params.replacementAssignments@@ -778,7 +810,9 @@ async function updateRuleWithRuleService(params: {projectId: params.projectId,ruleId: params.evaluationRuleId,targetObject:- "target" in params.input ? params.fields.targetObject : undefined,+ params.updatesFilterOrTarget && "target" in params.input+ ? params.fields.targetObject+ : undefined,name: params.input.name,enabled:params.input.enabled === undefined@@ -786,10 +820,9 @@ async function updateRuleWithRuleService(params: {: params.effectiveAssignmentCount > 0 &¶ms.fields.status === JobConfigState.ACTIVE,sampling: params.input.sampling,- filter:- "filter" in params.input || "target" in params.input- ? (params.fields.filter as FilterState)- : undefined,+ filter: params.updatesFilterOrTarget+ ? (params.fields.filter as FilterState)+ : undefined,evaluatorMappings,}),);@@ -831,7 +864,7 @@ export async function deletePublicEvaluationRule(params: {auditScope?: Pick<ApiAccessScope, "orgId" | "apiKeyId">;}) {const existing = await findPublicWritableV2EvaluationRuleOrThrow(params);- const existingPublic = toApiWritableV2EvaluationRule(existing);+ const existingPublic = toApiReadableV2EvaluationRule(existing);await runRuleServiceMutation(() =>ruleService.delete(params.projectId, params.evaluationRuleId),);diff --git a/web/src/features/evals/server/unstable-public-api/types.ts b/web/src/features/evals/server/unstable-public-api/types.tsindex 0f6c779521..0f7ed7e738 100644--- a/web/src/features/evals/server/unstable-public-api/types.ts+++ b/web/src/features/evals/server/unstable-public-api/types.ts@@ -10,6 +10,7 @@ import type {PublicEvaluationRuleEvaluatorReferenceType,PublicEvaluationRuleEvaluatorType,PublicEvaluationRuleFilterType,+ PublicEvaluationRuleReadFilterType,PromptVariableMappingReadType,PublicEvaluationRuleStatusType,PublicEvaluationRuleTargetType,@@ -104,8 +105,15 @@ type ApiLegacyEvaluationRuleRecord = ApiEvaluationRuleRecordBase & {mapping: LegacyPromptVariableMappingType[];};+type ApiReadableV2EvaluationRuleRecord = Omit<+ ApiWritableEvaluationRuleRecord,+ "filter"+> & {+ filter: PublicEvaluationRuleReadFilterType[];+};+export type ApiEvaluationRuleRecord =- | ApiWritableEvaluationRuleRecord+ | ApiReadableV2EvaluationRuleRecord| ApiLegacyEvaluationRuleRecord;export type EvaluationRuleEvaluatorFamilyReference =diff --git a/web/src/features/public-api/types/unstable-evaluation-rules.ts b/web/src/features/public-api/types/unstable-evaluation-rules.tsindex 69fa6d6fb7..56e18cb31c 100644--- a/web/src/features/public-api/types/unstable-evaluation-rules.ts+++ b/web/src/features/public-api/types/unstable-evaluation-rules.ts@@ -8,6 +8,7 @@ import {ObservationPromptVariableMappingInput,PUBLIC_EVALUATOR_TYPE_CODE,PublicEvaluationRuleFilter,+ PublicEvaluationRuleReadFilter,PublicEvaluationRuleEvaluator,PublicEvaluationRuleEvaluatorReference,PublicEvaluationRuleEvaluatorReferencePatch,@@ -48,6 +49,10 @@ const APIEvaluationRule = z}).strict();+const APIReadableV2EvaluationRule = APIEvaluationRule.extend({+ filter: z.array(PublicEvaluationRuleReadFilter),+});+const APILegacyEvaluationRule = z.object({...APIEvaluationRuleBase,@@ -66,7 +71,7 @@ const APILegacyEvaluationRule = z.strict();const APIReadableEvaluationRule = z.union([- APIEvaluationRule,+ APIReadableV2EvaluationRule,APILegacyEvaluationRule,]);@@ -237,7 +242,7 @@ export type PatchUnstableEvaluationRuleBodyType = z.infer<>;/** @alias */-export const PatchUnstableEvaluationRuleResponse = APIEvaluationRule;+export const PatchUnstableEvaluationRuleResponse = APIReadableV2EvaluationRule;/** @alias */export const DeleteUnstableEvaluationRuleQuery = GetUnstableEvaluationRuleQuery;diff --git a/web/src/features/public-api/types/unstable-public-evals-contract.ts b/web/src/features/public-api/types/unstable-public-evals-contract.tsindex f2ffaaa6be..a172d6e62e 100644--- a/web/src/features/public-api/types/unstable-public-evals-contract.ts+++ b/web/src/features/public-api/types/unstable-public-evals-contract.ts@@ -187,6 +187,20 @@ export const LegacyPromptVariableMapping = z},);+const PublicEvaluationRuleReadFilterBase = z+ .object({+ type: z.string(),+ column: z.string(),+ operator: z.string(),+ value: z.unknown().optional(),+ })+ .loose();++export const PublicEvaluationRuleReadFilter = z.union([+ PublicEvaluationRuleReadFilterBase.safeExtend({ key: z.string() }),+ PublicEvaluationRuleReadFilterBase,+]);+const filterSchemaFactories = {datetime: (columnId: string) =>timeFilter.safeExtend({ column: z.literal(columnId) }),@@ -305,6 +319,9 @@ export type LegacyPromptVariableMappingType = z.infer<export type PublicEvaluationRuleFilterType = z.infer<typeof PublicEvaluationRuleFilter>;+export type PublicEvaluationRuleReadFilterType = z.infer<+ typeof PublicEvaluationRuleReadFilter+>;export const UnstablePublicApiPaginationQuery = z.object({page: z.preprocess((x) => (x === "" ? undefined : x),