Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(api): keep historical evaluation rule filters readable (#16819)

Tobias Wochinger4e0ce09bbef6
diff --git a/fern/apis/server/definition/unstable/evaluation-rules.yml b/fern/apis/server/definition/unstable/evaluation-rules.yml
index 26240d0f13..a347e1e506 100644
--- a/fern/apis/server/definition/unstable/evaluation-rules.yml
+++ b/fern/apis/server/definition/unstable/evaluation-rules.yml
@@ -2,6 +2,7 @@
imports:
commons: ./commons.yml
errors: ./errors.yml
+ evals: ../evaluation-commons.yml
pagination: ../utils/pagination.yml
service:
@@ -323,7 +324,7 @@ service:
type: string
docs: Evaluation rule identifier.
request: UpdateEvaluationRuleRequest
- response: EvaluationRule
+ response: ReadableV2EvaluationRule
errors:
- errors.BadRequestError
- errors.UnauthorizedError
@@ -431,6 +432,22 @@ types:
mapping:
type: list<commons.PromptVariableMappingRead>
docs: Deprecated compatibility alias containing the effective mapping for `evaluators[0]`.
+
+ ReadableV2EvaluationRule:
+ extends: EvaluationRuleBase
+ properties:
+ evaluators:
+ type: list<EvaluationRuleEvaluatorAssignment>
+ docs: Evaluators attached to this rule in deterministic assignment order. A `null` mapping inherits the evaluator version's default mapping.
+ target:
+ type: commons.EvaluationRuleTarget
+ docs: Target object type that should trigger scoring.
+ filter:
+ type: list<evals.EvaluationRuleReadFilter>
+ docs: List of stored filter conditions returned verbatim. Historical filters remain readable even when the current write contract no longer accepts their shape.
+ mapping:
+ type: list<commons.PromptVariableMappingRead>
+ docs: Deprecated compatibility alias containing the effective mapping for `evaluators[0]`.
examples:
- value:
id: erule_123
@@ -498,7 +515,7 @@ types:
docs: Evaluation rule returned by list and get, including read-only legacy trace and dataset rules.
discriminated: false
union:
- - EvaluationRule
+ - ReadableV2EvaluationRule
- LegacyEvaluationRule
EvaluationRules:
diff --git a/web/public/generated/api/openapi.yml b/web/public/generated/api/openapi.yml
index e6541a1766..f5e6233254 100644
--- a/web/public/generated/api/openapi.yml
+++ b/web/public/generated/api/openapi.yml
@@ -8973,7 +8973,7 @@ paths:
content:
application/json:
schema:
- $ref: '#/components/schemas/unstableEvaluationRule'
+ $ref: '#/components/schemas/unstableReadableV2EvaluationRule'
'400':
description: ''
content:
@@ -19090,6 +19090,42 @@ components:
- mapping
allOf:
- $ref: '#/components/schemas/unstableEvaluationRuleBase'
+ unstableReadableV2EvaluationRule:
+ title: unstableReadableV2EvaluationRule
+ type: object
+ properties:
+ evaluators:
+ type: array
+ items:
+ $ref: '#/components/schemas/unstableEvaluationRuleEvaluatorAssignment'
+ description: >-
+ Evaluators attached to this rule in deterministic assignment order.
+ A `null` mapping inherits the evaluator version's default mapping.
+ target:
+ $ref: '#/components/schemas/unstableEvaluationRuleTarget'
+ description: Target object type that should trigger scoring.
+ filter:
+ type: array
+ items:
+ $ref: '#/components/schemas/EvaluationRuleReadFilter'
+ description: >-
+ List of stored filter conditions returned verbatim. Historical
+ filters remain readable even when the current write contract no
+ longer accepts their shape.
+ mapping:
+ type: array
+ items:
+ $ref: '#/components/schemas/unstablePromptVariableMappingRead'
+ description: >-
+ Deprecated compatibility alias containing the effective mapping for
+ `evaluators[0]`.
+ required:
+ - evaluators
+ - target
+ - filter
+ - mapping
+ allOf:
+ - $ref: '#/components/schemas/unstableEvaluationRuleBase'
unstableLegacyEvaluationRule:
title: unstableLegacyEvaluationRule
type: object
@@ -19144,7 +19180,7 @@ components:
unstableReadableEvaluationRule:
title: unstableReadableEvaluationRule
oneOf:
- - $ref: '#/components/schemas/unstableEvaluationRule'
+ - $ref: '#/components/schemas/unstableReadableV2EvaluationRule'
- $ref: '#/components/schemas/unstableLegacyEvaluationRule'
description: >-
Evaluation rule returned by list and get, including read-only legacy
diff --git a/web/src/__tests__/server/unit/unstable-evals-service.servertest.ts b/web/src/__tests__/server/unit/unstable-evals-service.servertest.ts
index 4e0dd8c518..f1918047a1 100644
--- a/web/src/__tests__/server/unit/unstable-evals-service.servertest.ts
+++ b/web/src/__tests__/server/unit/unstable-evals-service.servertest.ts
@@ -207,10 +207,16 @@ function buildMigratedRuleWithLegacyFilter() {
targetObject: "event",
filter: [
{
- type: "stringOptions",
- column: "experimentDatasetId",
- operator: "any of",
- value: ["legacy-value"],
+ type: "string",
+ column: "environment",
+ operator: "does not contain",
+ value: "langfuse-",
+ },
+ {
+ type: "null",
+ column: "experimentId",
+ operator: "is null",
+ value: "",
},
],
sampling: 1,
@@ -438,10 +444,16 @@ describe("unstable public evaluation-rule service", () => {
id: "rule",
filter: [
{
- type: "stringOptions",
- column: "datasetId",
- operator: "any of",
- value: ["legacy-value"],
+ type: "string",
+ column: "environment",
+ operator: "does not contain",
+ value: "langfuse-",
+ },
+ {
+ type: "null",
+ column: "experimentId",
+ operator: "is null",
+ value: "",
},
],
},
@@ -464,11 +476,11 @@ describe("unstable public evaluation-rule service", () => {
orgId: "org",
projectId: "project",
evaluationRuleId: "rule",
- input: { name: "Renamed rule" },
+ input: { name: "Renamed rule", target: "observation" },
}),
).resolves.toMatchObject({
name: "Renamed rule",
- filter: [{ column: "datasetId" }],
+ filter: [{ column: "environment" }, { column: "experimentId" }],
});
expect(prisma.evaluationRule.update).toHaveBeenCalledOnce();
});
@@ -506,6 +518,26 @@ describe("unstable public evaluation-rule service", () => {
expect(prisma.evaluationRule.update).toHaveBeenCalledOnce();
});
+ it("requires a replacement filter when changing the target of a migrated rule", async () => {
+ const migratedRule = buildMigratedRuleWithLegacyFilter();
+ (findPublicV2EvaluationRule as Mock).mockResolvedValue(migratedRule);
+ (prisma.evaluationRule.findFirst as Mock).mockResolvedValue(migratedRule);
+
+ await expect(
+ updatePublicEvaluationRule({
+ orgId: "org",
+ projectId: "project",
+ evaluationRuleId: "rule",
+ input: { target: "experiment" },
+ }),
+ ).rejects.toMatchObject({
+ httpCode: 400,
+ code: "invalid_body",
+ details: { field: "filter" },
+ });
+ expect(prisma.evaluationRule.update).not.toHaveBeenCalled();
+ });
+
it.each([
{
operation: "disabled",
diff --git a/web/src/features/evals/server/unstable-public-api/adapters.ts b/web/src/features/evals/server/unstable-public-api/adapters.ts
index 1cbb70820a..76aed05d5e 100644
--- a/web/src/features/evals/server/unstable-public-api/adapters.ts
+++ b/web/src/features/evals/server/unstable-public-api/adapters.ts
@@ -28,6 +28,7 @@ import {
PUBLIC_EVALUATOR_TYPE_CODE,
PUBLIC_EVALUATOR_TYPE_LLM_AS_JUDGE,
PublicEvaluationRuleFilter,
+ PublicEvaluationRuleReadFilter,
type PublicEvaluationRuleFilterType,
type PromptVariableMappingInputType,
type PromptVariableMappingReadType,
@@ -406,19 +407,9 @@ function toApiFilters(
target === "experiment"
? stripExperimentRootFilter(storedFilters.data)
: storedFilters.data;
- const publicFilters = effectiveFilters.map(toApiFilter);
- const parsedPublicFilters = z
- .array(PublicEvaluationRuleFilter)
- .safeParse(publicFilters);
-
- if (!parsedPublicFilters.success) {
- logger.error("Failed to parse unstable public evaluation rule filters", {
- issues: parsedPublicFilters.error.issues,
- });
- throw new InternalServerError("Evaluation rule filter is corrupted");
- }
-
- return parsedPublicFilters.data;
+ return z
+ .array(PublicEvaluationRuleReadFilter)
+ .parse(effectiveFilters.map(toApiFilter));
}
export function toApiEvaluator(params: {
@@ -614,7 +605,15 @@ export function toApiWritableV2EvaluationRule(
) {
throw new InternalServerError("Evaluation rule target is corrupted");
}
- return evaluationRule;
+
+ const filter = z
+ .array(PublicEvaluationRuleFilter)
+ .safeParse(evaluationRule.filter);
+ if (!filter.success) {
+ throw new InternalServerError("Evaluation rule filter is corrupted");
+ }
+
+ return { ...evaluationRule, filter: filter.data };
}
export function toEvaluationRuleInput(params: {
diff --git a/web/src/features/evals/server/unstable-public-api/evaluation-rule-service.ts b/web/src/features/evals/server/unstable-public-api/evaluation-rule-service.ts
index c5161c67ba..4cdf9f752c 100644
--- a/web/src/features/evals/server/unstable-public-api/evaluation-rule-service.ts
+++ b/web/src/features/evals/server/unstable-public-api/evaluation-rule-service.ts
@@ -2,6 +2,7 @@ import { type ApiAccessScope } from "@langfuse/shared/src/server";
import { EvalTemplateType, prisma } from "@langfuse/shared/src/db";
import {
JobConfigState,
+ InternalServerError,
LangfuseNotFoundError,
EvalTargetObject,
type FilterCondition,
@@ -23,6 +24,7 @@ import type {
PostUnstableEvaluationRuleBodyType,
} from "@/src/features/public-api/types/unstable-evaluation-rules";
import {
+ PublicEvaluationRuleFilter,
type PromptVariableMappingInputType,
type PromptVariableMappingReadType,
type PublicEvaluationRuleTargetType,
@@ -276,6 +278,17 @@ async function findPublicWritableV2EvaluationRuleOrThrow(params: {
return rule;
}
+function toApiReadableV2EvaluationRule(rule: StoredPublicV2EvaluationRule) {
+ const evaluationRule = toApiV2EvaluationRule(rule);
+ if (
+ evaluationRule.target !== "observation" &&
+ evaluationRule.target !== "experiment"
+ ) {
+ throw new InternalServerError("Evaluation rule target is corrupted");
+ }
+ return evaluationRule;
+}
+
export async function listPublicEvaluationRules(params: {
projectId: string;
page: number;
@@ -564,7 +577,7 @@ export async function updatePublicEvaluationRule(params: {
auditScope?: Pick<ApiAccessScope, "orgId" | "apiKeyId">;
}) {
const existing = await findPublicWritableV2EvaluationRuleOrThrow(params);
- const existingPublic = toApiWritableV2EvaluationRule(existing);
+ const existingPublic = toApiReadableV2EvaluationRule(existing);
const nextEnabled = params.input.enabled ?? existingPublic.enabled;
if (nextEnabled && existingPublic.status !== "active") {
await assertActivePublicApiEvaluationRuleLimitNotExceeded(params.projectId);
@@ -582,7 +595,24 @@ export async function updatePublicEvaluationRule(params: {
});
}
const updatesFilterOrTarget =
- suppliedFilter !== undefined || suppliedTarget !== undefined;
+ suppliedFilter !== undefined ||
+ (suppliedTarget !== undefined && suppliedTarget !== existingPublic.target);
+ const parsedExistingFilter = PublicEvaluationRuleFilter.array().safeParse(
+ existingPublic.filter,
+ );
+ if (
+ updatesFilterOrTarget &&
+ suppliedFilter === undefined &&
+ !parsedExistingFilter.success
+ ) {
+ throw createStructuredPublicApiError({
+ httpCode: 400,
+ code: "invalid_body",
+ message:
+ "The stored filter is no longer accepted for writes. Provide a target-compatible filter with this update.",
+ details: { field: "filter" },
+ });
+ }
const ruleFields = toEvaluationRuleFields({
name: params.input.name ?? existingPublic.name,
target: nextTarget,
@@ -590,7 +620,7 @@ export async function updatePublicEvaluationRule(params: {
sampling: params.input.sampling ?? existingPublic.sampling,
// Target changes must also validate the effective filter against the new target.
filter: updatesFilterOrTarget
- ? (suppliedFilter ?? existingPublic.filter)
+ ? (suppliedFilter ?? parsedExistingFilter.data ?? [])
: [],
});
@@ -705,8 +735,9 @@ export async function updatePublicEvaluationRule(params: {
replacementAssignments,
patchedFirstAssignment,
effectiveAssignmentCount: effectiveAssignments.length,
+ updatesFilterOrTarget,
});
- const evaluationRule = toApiWritableV2EvaluationRule(updated);
+ const evaluationRule = toApiReadableV2EvaluationRule(updated);
if (params.auditScope) {
await auditLog({
action: "update",
@@ -745,6 +776,7 @@ async function updateRuleWithRuleService(params: {
replacementAssignments: PreparedAssignment[] | null;
patchedFirstAssignment: PreparedAssignment | null;
effectiveAssignmentCount: number;
+ updatesFilterOrTarget: boolean;
}) {
const firstAssignment = params.existing.assignments[0];
const evaluatorMappings = params.replacementAssignments
@@ -778,7 +810,9 @@ async function updateRuleWithRuleService(params: {
projectId: params.projectId,
ruleId: params.evaluationRuleId,
targetObject:
- "target" in params.input ? params.fields.targetObject : undefined,
+ params.updatesFilterOrTarget && "target" in params.input
+ ? params.fields.targetObject
+ : undefined,
name: params.input.name,
enabled:
params.input.enabled === undefined
@@ -786,10 +820,9 @@ async function updateRuleWithRuleService(params: {
: params.effectiveAssignmentCount > 0 &&
params.fields.status === JobConfigState.ACTIVE,
sampling: params.input.sampling,
- filter:
- "filter" in params.input || "target" in params.input
- ? (params.fields.filter as FilterState)
- : undefined,
+ filter: params.updatesFilterOrTarget
+ ? (params.fields.filter as FilterState)
+ : undefined,
evaluatorMappings,
}),
);
@@ -831,7 +864,7 @@ export async function deletePublicEvaluationRule(params: {
auditScope?: Pick<ApiAccessScope, "orgId" | "apiKeyId">;
}) {
const existing = await findPublicWritableV2EvaluationRuleOrThrow(params);
- const existingPublic = toApiWritableV2EvaluationRule(existing);
+ const existingPublic = toApiReadableV2EvaluationRule(existing);
await runRuleServiceMutation(() =>
ruleService.delete(params.projectId, params.evaluationRuleId),
);
diff --git a/web/src/features/evals/server/unstable-public-api/types.ts b/web/src/features/evals/server/unstable-public-api/types.ts
index 0f6c779521..0f7ed7e738 100644
--- a/web/src/features/evals/server/unstable-public-api/types.ts
+++ b/web/src/features/evals/server/unstable-public-api/types.ts
@@ -10,6 +10,7 @@ import type {
PublicEvaluationRuleEvaluatorReferenceType,
PublicEvaluationRuleEvaluatorType,
PublicEvaluationRuleFilterType,
+ PublicEvaluationRuleReadFilterType,
PromptVariableMappingReadType,
PublicEvaluationRuleStatusType,
PublicEvaluationRuleTargetType,
@@ -104,8 +105,15 @@ type ApiLegacyEvaluationRuleRecord = ApiEvaluationRuleRecordBase & {
mapping: LegacyPromptVariableMappingType[];
};
+type ApiReadableV2EvaluationRuleRecord = Omit<
+ ApiWritableEvaluationRuleRecord,
+ "filter"
+> & {
+ filter: PublicEvaluationRuleReadFilterType[];
+};
+
export type ApiEvaluationRuleRecord =
- | ApiWritableEvaluationRuleRecord
+ | ApiReadableV2EvaluationRuleRecord
| ApiLegacyEvaluationRuleRecord;
export type EvaluationRuleEvaluatorFamilyReference =
diff --git a/web/src/features/public-api/types/unstable-evaluation-rules.ts b/web/src/features/public-api/types/unstable-evaluation-rules.ts
index 69fa6d6fb7..56e18cb31c 100644
--- a/web/src/features/public-api/types/unstable-evaluation-rules.ts
+++ b/web/src/features/public-api/types/unstable-evaluation-rules.ts
@@ -8,6 +8,7 @@ import {
ObservationPromptVariableMappingInput,
PUBLIC_EVALUATOR_TYPE_CODE,
PublicEvaluationRuleFilter,
+ PublicEvaluationRuleReadFilter,
PublicEvaluationRuleEvaluator,
PublicEvaluationRuleEvaluatorReference,
PublicEvaluationRuleEvaluatorReferencePatch,
@@ -48,6 +49,10 @@ const APIEvaluationRule = z
})
.strict();
+const APIReadableV2EvaluationRule = APIEvaluationRule.extend({
+ filter: z.array(PublicEvaluationRuleReadFilter),
+});
+
const APILegacyEvaluationRule = z
.object({
...APIEvaluationRuleBase,
@@ -66,7 +71,7 @@ const APILegacyEvaluationRule = z
.strict();
const APIReadableEvaluationRule = z.union([
- APIEvaluationRule,
+ APIReadableV2EvaluationRule,
APILegacyEvaluationRule,
]);
@@ -237,7 +242,7 @@ export type PatchUnstableEvaluationRuleBodyType = z.infer<
>;
/** @alias */
-export const PatchUnstableEvaluationRuleResponse = APIEvaluationRule;
+export const PatchUnstableEvaluationRuleResponse = APIReadableV2EvaluationRule;
/** @alias */
export const DeleteUnstableEvaluationRuleQuery = GetUnstableEvaluationRuleQuery;
diff --git a/web/src/features/public-api/types/unstable-public-evals-contract.ts b/web/src/features/public-api/types/unstable-public-evals-contract.ts
index f2ffaaa6be..a172d6e62e 100644
--- a/web/src/features/public-api/types/unstable-public-evals-contract.ts
+++ b/web/src/features/public-api/types/unstable-public-evals-contract.ts
@@ -187,6 +187,20 @@ export const LegacyPromptVariableMapping = z
},
);
+const PublicEvaluationRuleReadFilterBase = z
+ .object({
+ type: z.string(),
+ column: z.string(),
+ operator: z.string(),
+ value: z.unknown().optional(),
+ })
+ .loose();
+
+export const PublicEvaluationRuleReadFilter = z.union([
+ PublicEvaluationRuleReadFilterBase.safeExtend({ key: z.string() }),
+ PublicEvaluationRuleReadFilterBase,
+]);
+
const filterSchemaFactories = {
datetime: (columnId: string) =>
timeFilter.safeExtend({ column: z.literal(columnId) }),
@@ -305,6 +319,9 @@ export type LegacyPromptVariableMappingType = z.infer<
export type PublicEvaluationRuleFilterType = z.infer<
typeof PublicEvaluationRuleFilter
>;
+export type PublicEvaluationRuleReadFilterType = z.infer<
+ typeof PublicEvaluationRuleReadFilter
+>;
export const UnstablePublicApiPaginationQuery = z.object({
page: z.preprocess(
(x) => (x === "" ? undefined : x),