Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(evals): add facet type, built-in protection, and rule idle time (#18144)

Lysander Kieselcd8f35021f5f
diff --git a/packages/shared/prisma/migrations/20261001160000_add_facet_evaluator_type/migration.sql b/packages/shared/prisma/migrations/20261001160000_add_facet_evaluator_type/migration.sql
new file mode 100644
index 0000000000..c013294abe
--- /dev/null
+++ b/packages/shared/prisma/migrations/20261001160000_add_facet_evaluator_type/migration.sql
@@ -0,0 +1,7 @@
+ALTER TYPE "EvalTemplateType" ADD VALUE IF NOT EXISTS 'FACET';
+
+ALTER TABLE "evaluators"
+ ADD COLUMN IF NOT EXISTS "is_built_in" BOOLEAN NOT NULL DEFAULT FALSE;
+
+ALTER TABLE "evaluation_rules"
+ ADD COLUMN IF NOT EXISTS "idle_time" INTEGER;
diff --git a/packages/shared/prisma/schema.prisma b/packages/shared/prisma/schema.prisma
index ac463e5545..36735c43ad 100644
--- a/packages/shared/prisma/schema.prisma
+++ b/packages/shared/prisma/schema.prisma
@@ -1043,6 +1043,7 @@ model Evaluator {
name String
type EvalTemplateType
+ isBuiltIn Boolean @default(false) @map("is_built_in")
description String?
createdByUserId String? @map("created_by_user_id")
createdByUser User? @relation("EvaluatorCreatedByUser", fields: [createdByUserId], references: [id], onDelete: SetNull)
@@ -1089,6 +1090,7 @@ enum EvalTemplateType {
LLM_AS_JUDGE
CODE
DECISION_MODEL
+ FACET
}
enum EvalTemplateSourceCodeLanguage {
@@ -1167,6 +1169,7 @@ model EvaluationRule {
filter Json
sampling Decimal
delay Int
+ idleTime Int? @map("idle_time") // milliseconds
timeScope String[] @default(["NEW"]) @map("time_scope")
assignments EvaluationRuleEvaluatorAssignment[]
diff --git a/packages/shared/src/features/evals/types.ts b/packages/shared/src/features/evals/types.ts
index d8b572d9ea..47d0651b6d 100644
--- a/packages/shared/src/features/evals/types.ts
+++ b/packages/shared/src/features/evals/types.ts
@@ -16,6 +16,7 @@ export const EvalTemplateTypeEnum = {
LLM_AS_JUDGE: "LLM_AS_JUDGE",
CODE: "CODE",
DECISION_MODEL: "DECISION_MODEL",
+ FACET: "FACET",
} as const satisfies Record<EvalTemplateType, EvalTemplateType>;
export const EvalTemplateSourceCodeLanguageEnum = {
diff --git a/web/src/__tests__/server/evals-trpc.servertest.ts b/web/src/__tests__/server/evals-trpc.servertest.ts
index 19623ac893..635191e08d 100644
--- a/web/src/__tests__/server/evals-trpc.servertest.ts
+++ b/web/src/__tests__/server/evals-trpc.servertest.ts
@@ -114,6 +114,76 @@ describe("legacy evaluator compatibility service", () => {
expect(config?.evalTemplate?.name).toBe("Legacy score");
});
+ it("keeps facet rules out of legacy evaluator reads and writes", async () => {
+ const { project, service } = await prepare();
+ const rule = await createLegacyRule(project.id);
+ await prisma.evaluator.updateMany({
+ where: { projectId: project.id },
+ data: { type: EvalTemplateType.FACET, isBuiltIn: true },
+ });
+
+ await expect(service.counts(project.id)).resolves.toEqual({
+ configCount: 0,
+ configActiveCount: 0,
+ templateCount: 0,
+ legacyConfigCount: 0,
+ });
+ await expect(
+ service.listConfigs({ projectId: project.id }),
+ ).resolves.toEqual({
+ configs: [],
+ totalCount: 0,
+ });
+ await expect(service.getConfig(project.id, rule.id)).resolves.toBeNull();
+ await expect(
+ service.listConfigsByTemplateName(project.id, "Legacy score"),
+ ).resolves.toEqual([]);
+ await expect(
+ service.updateConfig({
+ projectId: project.id,
+ ruleId: rule.id,
+ data: { status: JobConfigState.INACTIVE },
+ }),
+ ).rejects.toThrow("Evaluation rule not found");
+ await expect(service.deleteConfig(project.id, rule.id)).resolves.toBe(
+ false,
+ );
+ });
+
+ it("rejects activation when a built-in evaluator remains blocked", async () => {
+ const { project, service } = await prepare();
+ const rule = await createLegacyRule(project.id);
+ await prisma.evaluationRule.update({
+ where: { id: rule.id },
+ data: { status: JobConfigState.INACTIVE },
+ });
+ await prisma.evaluator.updateMany({
+ where: { projectId: project.id },
+ data: { isBuiltIn: true, blockedAt: new Date() },
+ });
+
+ await expect(
+ service.updateConfig({
+ projectId: project.id,
+ ruleId: rule.id,
+ data: { status: JobConfigState.ACTIVE },
+ }),
+ ).rejects.toThrow("Blocked built-in evaluators cannot be activated");
+ await expect(
+ service.setConfigStatuses({
+ projectId: project.id,
+ ruleIds: [rule.id],
+ status: JobConfigState.ACTIVE,
+ }),
+ ).rejects.toThrow("Blocked built-in evaluators cannot be activated");
+ await expect(
+ prisma.evaluationRule.findUnique({
+ where: { id: rule.id },
+ select: { status: true },
+ }),
+ ).resolves.toEqual({ status: JobConfigState.INACTIVE });
+ });
+
it("projects an unassigned legacy-target rule for its read-only view", async () => {
const { project, service } = await prepare();
const rule = await createLegacyRule(project.id);
diff --git a/web/src/__tests__/server/evaluation-rule-v2-repository.servertest.ts b/web/src/__tests__/server/evaluation-rule-v2-repository.servertest.ts
index 13721e1928..a0d647d2f2 100644
--- a/web/src/__tests__/server/evaluation-rule-v2-repository.servertest.ts
+++ b/web/src/__tests__/server/evaluation-rule-v2-repository.servertest.ts
@@ -1,5 +1,9 @@
import { randomUUID } from "node:crypto";
-import { EvalTargetObject, type FilterState } from "@langfuse/shared";
+import {
+ EvalTargetObject,
+ EvalTemplateType,
+ type FilterState,
+} from "@langfuse/shared";
import { Prisma, prisma } from "@langfuse/shared/src/db";
import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
@@ -99,6 +103,42 @@ async function createRule({
}
describe("evaluation rule v2 repository", () => {
+ it("excludes facet rules from user-facing reads and mutations", async () => {
+ const evaluator = await createEvaluator();
+ await prisma.evaluator.update({
+ where: { id: evaluator.id },
+ data: { type: EvalTemplateType.FACET, isBuiltIn: true },
+ });
+ const rule = await createRule({ evaluatorId: evaluator.id });
+
+ await expect(
+ ruleRepository.listRules({
+ prisma,
+ input: { projectId, page: 1, limit: 50 },
+ }),
+ ).resolves.toEqual({ rules: [], totalItems: 0 });
+ await expect(
+ ruleRepository.findRule({ prisma, projectId, ruleId: rule.id }),
+ ).resolves.toBeNull();
+ await expect(
+ ruleRepository.listSelectedRuleIds({
+ prisma,
+ input: { projectId, ruleIds: [rule.id] },
+ }),
+ ).resolves.toEqual([]);
+ await expect(
+ ruleRepository.setRuleStatus({
+ prisma,
+ projectId,
+ ruleIds: [rule.id],
+ enabled: false,
+ }),
+ ).resolves.toEqual({ count: 0 });
+ await expect(
+ ruleRepository.deleteRule({ prisma, projectId, ruleId: rule.id }),
+ ).resolves.toBe(false);
+ });
+
describe("listRules", () => {
it("returns an empty list", async () => {
await expect(
diff --git a/web/src/__tests__/server/unit/batch-actions-comment-filter.servertest.ts b/web/src/__tests__/server/unit/batch-actions-comment-filter.servertest.ts
index ad04d2dc3b..a26560e397 100644
--- a/web/src/__tests__/server/unit/batch-actions-comment-filter.servertest.ts
+++ b/web/src/__tests__/server/unit/batch-actions-comment-filter.servertest.ts
@@ -311,6 +311,7 @@ describe("batched evaluation version selection", () => {
where: {
id: { in: [evaluatorId] },
projectId,
+ type: { not: "FACET" },
assignments: {
none: {
evaluationRule: {
diff --git a/web/src/features/evals/server/legacyCompatibilityService.ts b/web/src/features/evals/server/legacyCompatibilityService.ts
index dbbc97336a..f3e67e96a5 100644
--- a/web/src/features/evals/server/legacyCompatibilityService.ts
+++ b/web/src/features/evals/server/legacyCompatibilityService.ts
@@ -61,6 +61,39 @@ const LEGACY_TARGET_OBJECTS = [
EvalTargetObject.EXPERIMENT,
];
+const visibleRuleWhere = {
+ assignments: { none: { evaluator: { type: EvalTemplateType.FACET } } },
+} satisfies Prisma.EvaluationRuleWhereInput;
+
+async function assertNoBlockedBuiltInEvaluators(
+ tx: TransactionClient,
+ projectId: string,
+ ruleIds: string[],
+) {
+ const blockedRule = await tx.evaluationRule.findFirst({
+ where: {
+ projectId,
+ id: { in: ruleIds },
+ AND: [
+ visibleRuleWhere,
+ {
+ assignments: {
+ some: {
+ evaluator: { isBuiltIn: true, blockedAt: { not: null } },
+ },
+ },
+ },
+ ],
+ },
+ select: { id: true },
+ });
+ if (blockedRule) {
+ throw new InvalidRequestError(
+ "Blocked built-in evaluators cannot be activated",
+ );
+ }
+}
+
const latestVersion = {
orderBy: { version: "desc" as const },
take: 1,
@@ -250,6 +283,7 @@ function definitionFromEvaluator(
): EvaluatorDefinition | null {
const version = evaluator.versions[0];
if (!version) return null;
+ if (evaluator.type === EvalTemplateType.FACET) return null;
if (evaluator.type === EvalTemplateType.CODE) {
if (!version.sourceCode || !version.sourceCodeLanguage) return null;
return {
@@ -389,7 +423,10 @@ async function findReusableEvaluatorId(params: {
}
const version = await tx.evaluatorVersion.findFirst({
- where: { id: templateId, evaluator: { projectId } },
+ where: {
+ id: templateId,
+ evaluator: { projectId, type: { not: EvalTemplateType.FACET } },
+ },
include: { evaluator: { include: { versions: latestVersion } } },
});
if (!version) return null;
@@ -410,6 +447,7 @@ function isRunnableTemplate(template: {
type: EvalTemplateType;
sourceCodeLanguage: EvalTemplateSourceCodeLanguage | null;
}) {
+ if (template.type === EvalTemplateType.FACET) return false;
if (template.type !== EvalTemplateType.CODE) return true;
return (
@@ -497,7 +535,12 @@ async function clearEvaluatorBlock(params: {
if (!assignment) return;
await params.tx.evaluator.updateMany({
- where: { id: assignment.evaluatorId, projectId: params.projectId },
+ where: {
+ id: assignment.evaluatorId,
+ projectId: params.projectId,
+ isBuiltIn: false,
+ type: { not: EvalTemplateType.FACET },
+ },
data: resetEvalConfigBlockFields,
});
}
@@ -523,10 +566,13 @@ async function applyScoreNameChange(params: {
},
});
if (!evaluator || evaluator.name === scoreName) return;
+ if (evaluator.isBuiltIn || evaluator.type === EvalTemplateType.FACET) {
+ throw new InvalidRequestError("Built-in evaluators cannot be edited");
+ }
if (evaluator._count.assignments <= 1) {
await tx.evaluator.update({
- where: { id: evaluatorId, projectId },
+ where: { id: evaluatorId, projectId, isBuiltIn: false },
data: { name: scoreName },
});
return;
@@ -611,6 +657,7 @@ const legacyConfigIdsQuery = (params: {
) a ON a."evaluation_rule_id" = r."id"
JOIN "evaluators" e ON e."id" = a."evaluator_id"
WHERE r."project_id" = ${params.projectId}
+ AND e."type"::text <> ${EvalTemplateType.FACET}
) jc
WHERE TRUE
${params.targetCondition}
@@ -647,14 +694,23 @@ export class LegacyEvalCompatibilityService {
async counts(projectId: string) {
const [configCount, configActiveCount, templateCount, legacyConfigCount] =
await Promise.all([
- this.prisma.evaluationRule.count({ where: { projectId } }),
this.prisma.evaluationRule.count({
- where: { projectId, status: JobConfigState.ACTIVE },
+ where: { projectId, ...visibleRuleWhere },
}),
- this.prisma.evaluator.count({ where: { projectId } }),
this.prisma.evaluationRule.count({
where: {
projectId,
+ ...visibleRuleWhere,
+ status: JobConfigState.ACTIVE,
+ },
+ }),
+ this.prisma.evaluator.count({
+ where: { projectId, type: { not: EvalTemplateType.FACET } },
+ }),
+ this.prisma.evaluationRule.count({
+ where: {
+ projectId,
+ ...visibleRuleWhere,
targetObject: {
in: [EvalTargetObject.TRACE, EvalTargetObject.DATASET],
},
@@ -757,7 +813,7 @@ export class LegacyEvalCompatibilityService {
async getConfig(projectId: string, ruleId: string) {
const rule = await this.prisma.evaluationRule.findFirst({
- where: { id: ruleId, projectId },
+ where: { id: ruleId, projectId, ...visibleRuleWhere },
include: ruleInclude,
});
if (!rule || rule.assignments.length > 1) return null;
@@ -811,7 +867,7 @@ export class LegacyEvalCompatibilityService {
options: { collapseManagedCopies?: boolean } = {},
) {
const evaluators = await this.prisma.evaluator.findMany({
- where: { projectId },
+ where: { projectId, type: { not: EvalTemplateType.FACET } },
include: { versions: latestVersion },
orderBy: [{ name: "asc" }, { createdAt: "asc" }],
});
@@ -835,6 +891,7 @@ export class LegacyEvalCompatibilityService {
const evaluators = await this.prisma.evaluator.findMany({
where: {
projectId: params.projectId,
+ type: { not: EvalTemplateType.FACET },
...(search
? { name: { contains: search, mode: "insensitive" as const } }
: {}),
@@ -947,7 +1004,10 @@ export class LegacyEvalCompatibilityService {
return isRunnableTemplate(legacyTemplate) ? legacyTemplate : null;
}
const version = await this.prisma.evaluatorVersion.findFirst({
- where: { id: templateId, evaluator: { projectId } },
+ where: {
+ id: templateId,
+ evaluator: { projectId, type: { not: EvalTemplateType.FACET } },
+ },
include: { evaluator: true },
});
if (!version) return null;
@@ -979,7 +1039,10 @@ export class LegacyEvalCompatibilityService {
const version = await this.prisma.evaluatorVersion.findFirst({
where: {
id: params.templateId,
- evaluator: { projectId: params.projectId },
+ evaluator: {
+ projectId: params.projectId,
+ type: { not: EvalTemplateType.FACET },
+ },
},
include: { evaluator: { include: { versions: latestVersion } } },
});
@@ -1048,6 +1111,7 @@ export class LegacyEvalCompatibilityService {
where: {
id: params.reuseEvaluatorFromRuleId,
projectId: params.projectId,
+ ...visibleRuleWhere,
targetObject: { in: LEGACY_TARGET_OBJECTS },
},
select: {
@@ -1154,7 +1218,11 @@ export class LegacyEvalCompatibilityService {
const source = await tx.evaluatorVersion.findFirst({
where: {
id: params.intent.sourceTemplateId,
- evaluator: { projectId: params.projectId },
+ evaluator: {
+ projectId: params.projectId,
+ isBuiltIn: false,
+ type: { not: EvalTemplateType.FACET },
+ },
},
select: { evaluatorId: true },
});
@@ -1168,6 +1236,9 @@ export class LegacyEvalCompatibilityService {
where: { id: source.evaluatorId, projectId: params.projectId },
include: { versions: latestVersion },
});
+ if (evaluator.isBuiltIn) {
+ throw new InvalidRequestError("Built-in evaluators cannot be edited");
+ }
if (
evaluator.name !== params.name ||
evaluator.type !== params.definition.type
@@ -1309,7 +1380,7 @@ export class LegacyEvalCompatibilityService {
async listTemplateVersions(projectId: string, name: string) {
const evaluators = await this.prisma.evaluator.findMany({
- where: { projectId, name },
+ where: { projectId, name, type: { not: EvalTemplateType.FACET } },
include: { versions: { orderBy: { version: "desc" } } },
});
return evaluators.flatMap((evaluator) =>
@@ -1327,7 +1398,10 @@ export class LegacyEvalCompatibilityService {
const rules = await this.prisma.evaluationRule.findMany({
where: {
projectId,
- assignments: { some: { evaluator: { name } } },
+ AND: [
+ visibleRuleWhere,
+ { assignments: { some: { evaluator: { name } } } },
+ ],
},
include: ruleInclude,
});
@@ -1354,6 +1428,7 @@ export class LegacyEvalCompatibilityService {
where: {
id: params.ruleId,
projectId: params.projectId,
+ ...visibleRuleWhere,
targetObject: { in: LEGACY_TARGET_OBJECTS },
},
include: { assignments: true },
@@ -1370,6 +1445,7 @@ export class LegacyEvalCompatibilityService {
});
}
if (params.data.status === JobConfigState.ACTIVE) {
+ await assertNoBlockedBuiltInEvaluators(tx, params.projectId, [rule.id]);
// Blocking moved to the evaluator, so activating the rule alone would
// leave the evaluator paused and the worker would keep skipping it.
await clearEvaluatorBlock({
@@ -1432,10 +1508,18 @@ export class LegacyEvalCompatibilityService {
if (params.ruleIds.length === 0) return 0;
return this.prisma.$transaction(async (tx) => {
+ if (params.status === JobConfigState.ACTIVE) {
+ await assertNoBlockedBuiltInEvaluators(
+ tx,
+ params.projectId,
+ params.ruleIds,
+ );
+ }
const { count } = await tx.evaluationRule.updateMany({
where: {
id: { in: params.ruleIds },
projectId: params.projectId,
+ ...visibleRuleWhere,
targetObject: { in: LEGACY_TARGET_OBJECTS },
},
data: { status: params.status },
@@ -1447,6 +1531,7 @@ export class LegacyEvalCompatibilityService {
where: {
projectId: params.projectId,
evaluationRuleId: { in: params.ruleIds },
+ evaluationRule: visibleRuleWhere,
},
select: { evaluatorId: true },
},
@@ -1455,6 +1540,8 @@ export class LegacyEvalCompatibilityService {
where: {
projectId: params.projectId,
id: { in: assignments.map(({ evaluatorId }) => evaluatorId) },
+ isBuiltIn: false,
+ type: { not: EvalTemplateType.FACET },
},
data: resetEvalConfigBlockFields,
});
@@ -1469,6 +1556,7 @@ export class LegacyEvalCompatibilityService {
where: {
id: ruleId,
projectId,
+ ...visibleRuleWhere,
targetObject: { in: LEGACY_TARGET_OBJECTS },
},
select: { id: true },
@@ -1493,7 +1581,10 @@ export class LegacyEvalCompatibilityService {
return [];
}
const version = await this.prisma.evaluatorVersion.findFirst({
- where: { id: templateId, evaluator: { projectId } },
+ where: {
+ id: templateId,
+ evaluator: { projectId, type: { not: EvalTemplateType.FACET } },
+ },
select: { evaluatorId: true },
});
if (!version) return [];
@@ -1517,7 +1608,14 @@ export class LegacyEvalCompatibilityService {
return this.prisma.$transaction(async (tx) => {
const version = await tx.evaluatorVersion.findFirst({
- where: { id: templateId, evaluator: { projectId } },
+ where: {
+ id: templateId,
+ evaluator: {
+ projectId,
+ isBuiltIn: false,
+ type: { not: EvalTemplateType.FACET },
+ },
+ },
select: { evaluatorId: true },
});
if (!version) throw new LangfuseNotFoundError("Evaluator not found");
@@ -1543,7 +1641,7 @@ export class LegacyEvalCompatibilityService {
where: { evaluatorId: version.evaluatorId },
});
await tx.evaluator.delete({
- where: { id: version.evaluatorId, projectId },
+ where: { id: version.evaluatorId, projectId, isBuiltIn: false },
});
return versions;
});
diff --git a/web/src/features/evals/v2/AGENTS.md b/web/src/features/evals/v2/AGENTS.md
index 77a0cdfb34..aeafd59a6b 100644
--- a/web/src/features/evals/v2/AGENTS.md
+++ b/web/src/features/evals/v2/AGENTS.md
@@ -12,6 +12,8 @@ The new data model is
- Rule (basically the old `job_configuration`)
- Rule assignments (association table handling the n:m relationship)
+FACET evaluators are internal and must be excluded from user-facing evaluator and rule queries.
+
Traces captured during the eval executions in the past only captured `job_configuration_id`.
Only new runs capture `evaluator_id` and `evaluation_rule_id`.
diff --git a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryMethodBadge/EvaluatorGalleryMethodBadge.tsx b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryMethodBadge/EvaluatorGalleryMethodBadge.tsx
index 129d111d87..b43e8fe81b 100644
--- a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryMethodBadge/EvaluatorGalleryMethodBadge.tsx
+++ b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryMethodBadge/EvaluatorGalleryMethodBadge.tsx
@@ -1,4 +1,4 @@
-import { Code2, Scale, Sparkles } from "lucide-react";
+import { Code2, Scale, Sparkles, Tags } from "lucide-react";
import { EvalTemplateTypeEnum, type EvalTemplateType } from "@langfuse/shared";
const METHOD_BADGES = {
@@ -11,6 +11,7 @@ const METHOD_BADGES = {
Icon: Scale,
label: "Decision model",
},
+ [EvalTemplateTypeEnum.FACET]: { Icon: Tags, label: "Facet" },
} as const satisfies Record<EvalTemplateType, { Icon: unknown; label: string }>;
export function EvaluatorGalleryMethodBadge({
diff --git a/web/src/features/evals/v2/components/EvaluatorTestPanel/components/TestSection/components/TestSectionContainer/TestSectionContainer.tsx b/web/src/features/evals/v2/components/EvaluatorTestPanel/components/TestSection/components/TestSectionContainer/TestSectionContainer.tsx
index b1ab135cb7..34b6eb61c9 100644
--- a/web/src/features/evals/v2/components/EvaluatorTestPanel/components/TestSection/components/TestSectionContainer/TestSectionContainer.tsx
+++ b/web/src/features/evals/v2/components/EvaluatorTestPanel/components/TestSection/components/TestSectionContainer/TestSectionContainer.tsx
@@ -12,7 +12,10 @@ import { TestRunCard } from "@/src/features/evals/v2/components/EvaluatorTestPan
import { toTestResultPanelState } from "@/src/features/evals/v2/fns/evaluatorTesting/toTestResultPanelState";
import type { EvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";
-const RESULT_TITLES: Record<EvalTemplateType, TestResultPanelTitle> = {
+const RESULT_TITLES: Record<
+ Exclude<EvalTemplateType, "FACET">,
+ TestResultPanelTitle
+> = {
LLM_AS_JUDGE: "LLM Output",
CODE: "Code Output",
DECISION_MODEL: "Decision Model Output",
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DefinitionStepContainer/DefinitionStepContainer.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DefinitionStepContainer/DefinitionStepContainer.tsx
index c1e0532d2a..39e2771ee8 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DefinitionStepContainer/DefinitionStepContainer.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DefinitionStepContainer/DefinitionStepContainer.tsx
@@ -53,6 +53,7 @@ export function DefinitionStepContainer({
})),
);
const changeType = (type: EvalTemplateType) => {
+ if (type === "FACET") return;
const previousEvaluatorType = store.getState().type;
state.actions.setType(type);
if (type !== previousEvaluatorType) {
diff --git a/web/src/features/evals/v2/fns/evaluators/evaluatorTypeLabel.ts b/web/src/features/evals/v2/fns/evaluators/evaluatorTypeLabel.ts
index 4deabacb3d..7a3ac04bb0 100644
--- a/web/src/features/evals/v2/fns/evaluators/evaluatorTypeLabel.ts
+++ b/web/src/features/evals/v2/fns/evaluators/evaluatorTypeLabel.ts
@@ -9,5 +9,7 @@ export function evaluatorTypeLabel(type: EvalTemplateType): string {
return "Decision model (experimental)";
case EvalTemplateTypeEnum.LLM_AS_JUDGE:
return "LLM as a judge";
+ case EvalTemplateTypeEnum.FACET:
+ return "Facet";
}
}
diff --git a/web/src/features/evals/v2/pages/EvaluatorDetailPage.tsx b/web/src/features/evals/v2/pages/EvaluatorDetailPage.tsx
index 51a9fa03c0..e272af5909 100644
--- a/web/src/features/evals/v2/pages/EvaluatorDetailPage.tsx
+++ b/web/src/features/evals/v2/pages/EvaluatorDetailPage.tsx
@@ -39,6 +39,8 @@ function toSetupDefinition(
sourceCode: latest.sourceCode ?? "",
sourceCodeLanguage: latest.sourceCodeLanguage ?? "TYPESCRIPT",
};
+ case "FACET":
+ throw new Error("Facets are not shown as evaluators");
}
}
@@ -54,7 +56,7 @@ export default function EvaluatorDetailPage() {
if (evaluator.isPending) {
return <EvaluatorSetupLoadingPage mode="edit" projectId={projectId} />;
}
- if (!evaluator.data?.versions[0]) {
+ if (!evaluator.data?.versions[0] || evaluator.data.type === "FACET") {
return <div className="p-6">Evaluator not found</div>;
}
diff --git a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
index 7c0bb5e926..a81fd9699c 100644
--- a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
+++ b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
@@ -77,7 +77,7 @@ type InitialEvaluator = {
id: string;
name: string;
description: string | null;
- type: EvalTemplateType;
+ type: Exclude<EvalTemplateType, "FACET">;
definition: NormalizedEvaluatorDefinition;
blockedAt: Date | null;
blockReason: EvaluatorBlockReason | null;
@@ -103,6 +103,9 @@ export function applyEvaluatorSuggestion(
export function getEvaluatorVersionDefinition(
version: EvaluatorVersion,
): NormalizedEvaluatorDefinition {
+ if (version.type === "FACET") {
+ throw new Error("Facets cannot be edited as evaluators");
+ }
if (version.type === "CODE") {
return {
type: version.type,
@@ -167,7 +170,7 @@ export function EvaluatorSetupPage(
mode: "create";
projectId: string;
initialDraft: EvaluatorSetupDraft | null;
- initialType: EvalTemplateType;
+ initialType: Exclude<EvalTemplateType, "FACET">;
creationSource: EvaluatorCreationSource;
}
| {
diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts b/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts
index 67f016ece9..60bc265154 100644
--- a/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts
+++ b/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts
@@ -45,6 +45,10 @@ const eventEvaluatorFilterColumnIds = new Set(
eventsEvalFilterColumns.map((column) => column.id),
);
+const visibleEvaluatorWhere = {
+ type: { not: EvalTemplateType.FACET },
+} as const satisfies Prisma.EvaluatorWhereInput;
+
const publicEvaluationRuleAssignments = (projectId: string) =>
({
where: { projectId },
@@ -58,6 +62,7 @@ const publicEvaluationRuleAssignments = (projectId: string) =>
}) satisfies Prisma.EvaluationRuleEvaluatorAssignmentFindManyArgs;
export const batchEligibleEvaluatorWhere = {
+ ...visibleEvaluatorWhere,
// Trace/dataset assignments carry rule-specific mappings that an
// observation batch cannot resolve when it addresses the evaluator alone.
assignments: {
@@ -188,6 +193,7 @@ async function evaluatorIdsMatchingModelFilters(params: {
LEFT JOIN default_llm_models AS default_model
ON default_model.project_id = evaluator.project_id
WHERE evaluator.project_id = ${params.projectId}
+ AND evaluator.type <> 'FACET'
AND ${Prisma.join(predicates, " AND ")}
`,
);
@@ -272,6 +278,7 @@ async function evaluatorWhere(params: {
return {
projectId: params.projectId,
+ ...visibleEvaluatorWhere,
...(modelEvaluatorIds ? { id: { in: modelEvaluatorIds } } : {}),
...(params.search
? { name: { contains: params.search, mode: "insensitive" as const } }
@@ -352,6 +359,7 @@ function getEffectiveModel(
case EvalTemplateType.DECISION_MODEL:
return evaluator.versions[0]?.model ?? null;
case EvalTemplateType.CODE:
+ case EvalTemplateType.FACET:
return null;
}
}
@@ -365,8 +373,10 @@ export async function listEvaluatorsCursor(params: {
types?: EvalTemplateType[];
}) {
const baseWhere: Prisma.EvaluatorWhereInput = {
- ...(await evaluatorWhere(params)),
- ...(params.types ? { type: { in: params.types } } : {}),
+ AND: [
+ await evaluatorWhere(params),
+ ...(params.types ? [{ type: { in: params.types } }] : []),
+ ],
};
const where: Prisma.EvaluatorWhereInput = params.cursor
? {
@@ -419,7 +429,7 @@ export async function listEvaluatorFilterOptions(params: {
}) {
const [evaluators, defaultModel] = await Promise.all([
params.prisma.evaluator.findMany({
- where: { projectId: params.projectId },
+ where: { projectId: params.projectId, ...visibleEvaluatorWhere },
select: {
name: true,
type: true,
@@ -477,7 +487,11 @@ export function countProjectEvaluators(params: {
evaluatorIds: string[];
}) {
return params.prisma.evaluator.count({
- where: { projectId: params.projectId, id: { in: params.evaluatorIds } },
+ where: {
+ projectId: params.projectId,
+ id: { in: params.evaluatorIds },
+ ...visibleEvaluatorWhere,
+ },
});
}
@@ -491,6 +505,7 @@ export async function listEvaluatorOptions(params: {
const evaluators = await params.prisma.evaluator.findMany({
where: {
projectId: params.projectId,
+ ...visibleEvaluatorWhere,
...(params.search
? { name: { contains: params.search, mode: "insensitive" } }
: {}),
@@ -530,7 +545,11 @@ export function findEvaluator(params: {
evaluatorId: string;
}) {
return params.prisma.evaluator.findFirst({
- where: { id: params.evaluatorId, projectId: params.projectId },
+ where: {
+ id: params.evaluatorId,
+ projectId: params.projectId,
+ ...visibleEvaluatorWhere,
+ },
include: {
versions: latestVersionWithCreator,
createdByUser,
@@ -581,6 +600,7 @@ export function findEvaluatorsByIds(params: {
where: {
id: { in: params.evaluatorIds },
projectId: params.projectId,
+ ...visibleEvaluatorWhere,
},
include: {
versions: latestVersion,
@@ -598,7 +618,7 @@ export async function listEvaluatorVersions(params: {
const versions = await params.prisma.evaluatorVersion.findMany({
where: {
evaluatorId: params.evaluatorId,
- evaluator: { projectId: params.projectId },
+ evaluator: { projectId: params.projectId, ...visibleEvaluatorWhere },
...(params.cursor ? { version: { lt: params.cursor } } : {}),
},
orderBy: { version: "desc" },
@@ -623,7 +643,11 @@ export function findEvaluatorsByName(params: {
name: string;
}) {
return params.prisma.evaluator.findMany({
- where: { projectId: params.projectId, name: params.name },
+ where: {
+ projectId: params.projectId,
+ name: params.name,
+ ...visibleEvaluatorWhere,
+ },
include: { versions: latestVersion },
take: 2,
});
@@ -675,7 +699,12 @@ export function blockEvaluator(params: {
message: string;
}) {
return params.tx.evaluator.update({
- where: { id: params.evaluatorId, projectId: params.projectId },
+ where: {
+ id: params.evaluatorId,
+ projectId: params.projectId,
+ isBuiltIn: false,
+ ...visibleEvaluatorWhere,
+ },
data: {
blockedAt: new Date(),
blockReason: params.reason,
@@ -690,7 +719,12 @@ export function unblockEvaluator(params: {
evaluatorId: string;
}) {
return params.tx.evaluator.update({
- where: { id: params.evaluatorId, projectId: params.projectId },
+ where: {
+ id: params.evaluatorId,
+ projectId: params.projectId,
+ isBuiltIn: false,
+ ...visibleEvaluatorWhere,
+ },
data: {
blockedAt: null,
blockReason: null,
@@ -707,7 +741,12 @@ export function updateEvaluatorMetadata(params: {
description?: string | null;
}) {
return params.tx.evaluator.update({
- where: { id: params.evaluatorId, projectId: params.projectId },
+ where: {
+ id: params.evaluatorId,
+ projectId: params.projectId,
+ isBuiltIn: false,
+ ...visibleEvaluatorWhere,
+ },
data: {
...(params.name === undefined ? {} : { name: params.name }),
...(params.description === undefined
@@ -757,7 +796,12 @@ export async function deleteEvaluator(params: {
select: { evaluationRuleId: true },
});
const result = await params.prisma.evaluator.deleteMany({
- where: { id: params.evaluatorId, projectId: params.projectId },
+ where: {
+ id: params.evaluatorId,
+ projectId: params.projectId,
+ isBuiltIn: false,
+ ...visibleEvaluatorWhere,
+ },
});
if (result.count > 0) {
await setRuleStatus({
diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorService.ts b/web/src/features/evals/v2/server/evaluators/evaluatorService.ts
index cee41c01ef..872a2aa68e 100644
--- a/web/src/features/evals/v2/server/evaluators/evaluatorService.ts
+++ b/web/src/features/evals/v2/server/evaluators/evaluatorService.ts
@@ -483,6 +483,7 @@ export class EvaluatorService {
evaluatorId,
});
if (!evaluator) throw new LangfuseNotFoundError("Evaluator not found");
+ assertEditableEvaluator(evaluator);
if (!evaluator.blockedAt)
return normalizeEvaluatorPromptMessages(evaluator);
@@ -631,7 +632,11 @@ export class EvaluatorService {
if (definition) {
if (evaluatorId) {
const evaluator = await this.prisma.evaluator.findFirst({
- where: { id: evaluatorId, projectId: params.projectId },
+ where: {
+ id: evaluatorId,
+ projectId: params.projectId,
+ type: { not: EvalTemplateType.FACET },
+ },
select: { id: true },
});
// The setup editor pre-generates a UUID so a test run can be attributed
@@ -729,10 +734,23 @@ async function deleteEvaluator(params: {
projectId: string;
evaluatorId: string;
}) {
+ const evaluator = await repository.findEvaluator({
+ prisma: params.prisma,
+ projectId: params.projectId,
+ evaluatorId: params.evaluatorId,
+ });
+ if (!evaluator) throw new LangfuseNotFoundError("Evaluator not found");
+ assertEditableEvaluator(evaluator);
const deleted = await repository.deleteEvaluator(params);
if (!deleted) throw new LangfuseNotFoundError("Evaluator not found");
}
+function assertEditableEvaluator(evaluator: { isBuiltIn: boolean }) {
+ if (evaluator.isBuiltIn) {
+ throw new InvalidRequestError("Built-in evaluators cannot be edited");
+ }
+}
+
async function patchEvaluator(params: {
tx: Prisma.TransactionClient;
input: Omit<PatchEvaluatorInput, "definition"> & {
@@ -748,6 +766,7 @@ async function patchEvaluator(params: {
evaluatorId: input.evaluatorId,
});
if (!current) throw new LangfuseNotFoundError("Evaluator not found");
+ assertEditableEvaluator(current);
if (input.definition && current.type !== input.definition.type) {
throw new LangfuseConflictError("Evaluator type cannot be changed");
}
@@ -811,6 +830,7 @@ async function updateEvaluator(params: {
evaluatorId: input.evaluatorId,
});
if (!current) throw new LangfuseNotFoundError("Evaluator not found");
+ assertEditableEvaluator(current);
if (current.type !== input.definition.type) {
throw new LangfuseConflictError("Evaluator type cannot be changed");
}
diff --git a/web/src/features/evals/v2/server/rules/ruleRepository.ts b/web/src/features/evals/v2/server/rules/ruleRepository.ts
index ac4e5c6804..8acd59cff6 100644
--- a/web/src/features/evals/v2/server/rules/ruleRepository.ts
+++ b/web/src/features/evals/v2/server/rules/ruleRepository.ts
@@ -6,6 +6,7 @@ import {
import {
coerceLegacyEmptyMetadataFilters,
EvalTargetObject,
+ EvalTemplateType,
type FilterState,
type ObservationVariableMapping,
} from "@langfuse/shared";
@@ -30,6 +31,10 @@ export type RulePrisma = PrismaClient | Prisma.TransactionClient;
const MAX_REUSABLE_FILTER_CANDIDATES = 20;
+const visibleRuleWhere = {
+ assignments: { none: { evaluator: { type: EvalTemplateType.FACET } } },
+} satisfies Prisma.EvaluationRuleWhereInput;
+
const latestVersion = {
orderBy: { version: "desc" as const },
take: 1,
@@ -110,6 +115,7 @@ function ruleWhere(params: {
return {
projectId: params.projectId,
+ ...visibleRuleWhere,
...(params.search
? { name: { contains: params.search, mode: "insensitive" as const } }
: {}),
@@ -200,6 +206,7 @@ export async function listReusableFilterCandidates(params: {
const rules = await params.prisma.evaluationRule.findMany({
where: {
projectId: params.projectId,
+ ...visibleRuleWhere,
targetObject: {
in: [EvalTargetObject.EVENT, EvalTargetObject.EXPERIMENT],
},
@@ -232,7 +239,7 @@ export async function listRuleFilterOptions(params: {
}) {
const [rules, upgradeRequiredRule] = await Promise.all([
params.prisma.evaluationRule.findMany({
- where: { projectId: params.projectId },
+ where: { projectId: params.projectId, ...visibleRuleWhere },
select: {
name: true,
createdByUser: { select: { id: true, name: true, email: true } },
@@ -241,6 +248,7 @@ export async function listRuleFilterOptions(params: {
params.prisma.evaluationRule.findFirst({
where: {
projectId: params.projectId,
+ ...visibleRuleWhere,
...upgradeRequiredRuleCondition,
},
select: { id: true },
@@ -267,7 +275,11 @@ export function findRule(params: {
ruleId: string;
}) {
return params.prisma.evaluationRule.findFirst({
- where: { id: params.ruleId, projectId: params.projectId },
+ where: {
+ id: params.ruleId,
+ projectId: params.projectId,
+ ...visibleRuleWhere,
+ },
include: ruleInclude,
});
}
@@ -281,6 +293,7 @@ export async function findActiveRuleWithMatchingFilterAndSampling(params: {
const rules = await params.prisma.evaluationRule.findMany({
where: {
projectId: params.projectId,
+ ...visibleRuleWhere,
status: JobConfigState.ACTIVE,
targetObject: EvalTargetObject.EVENT,
sampling: params.sampling,
@@ -340,7 +353,11 @@ export function updateRule(params: {
filter?: Prisma.InputJsonValue;
}) {
return params.prisma.evaluationRule.update({
- where: { id: params.input.ruleId, projectId: params.input.projectId },
+ where: {
+ id: params.input.ruleId,
+ projectId: params.input.projectId,
+ ...visibleRuleWhere,
+ },
data: {
...(params.targetObject === undefined
? {}
@@ -376,6 +393,7 @@ export function setRuleStatus(params: {
return params.prisma.evaluationRule.updateMany({
where: {
projectId: params.projectId,
+ ...visibleRuleWhere,
...(params.ruleIds === undefined ? {} : { id: { in: params.ruleIds } }),
...(params.unassignedOnly ? { assignments: { none: {} } } : {}),
...(params.sampling === undefined ? { status: { not: status } } : {}),
@@ -393,7 +411,11 @@ export async function deleteRule(params: {
ruleId: string;
}) {
const result = await params.prisma.evaluationRule.deleteMany({
- where: { id: params.ruleId, projectId: params.projectId },
+ where: {
+ id: params.ruleId,
+ projectId: params.projectId,
+ ...visibleRuleWhere,
+ },
});
if (result.count === 0) return false;
@@ -410,13 +432,28 @@ export async function deleteRules(params: {
projectId: string;
ruleIds: string[];
}) {
+ const visibleIds = await params.prisma.evaluationRule.findMany({
+ where: {
+ projectId: params.projectId,
+ id: { in: params.ruleIds },
+ ...visibleRuleWhere,
+ },
+ select: { id: true },
+ });
+ const ruleIds = visibleIds.map(({ id }) => id);
+ if (ruleIds.length === 0) return 0;
+
const result = await params.prisma.evaluationRule.deleteMany({
- where: { projectId: params.projectId, id: { in: params.ruleIds } },
+ where: {
+ projectId: params.projectId,
+ id: { in: ruleIds },
+ ...visibleRuleWhere,
+ },
});
await params.prisma.jobExecution.deleteMany({
where: {
projectId: params.projectId,
- jobConfigurationId: { in: params.ruleIds },
+ jobConfigurationId: { in: ruleIds },
},
});
return result.count;
@@ -431,6 +468,7 @@ export async function listSelectedRuleIds(params: {
where: {
projectId: params.input.projectId,
id: { in: params.input.ruleIds },
+ ...visibleRuleWhere,
},
select: { id: true },
});
@@ -529,6 +567,8 @@ export async function countRulesForEvaluators(params: {
where: {
projectId: params.projectId,
evaluatorId: { in: params.evaluatorIds },
+ evaluator: { type: { not: EvalTemplateType.FACET } },
+ evaluationRule: visibleRuleWhere,
},
_count: { _all: true },
});
@@ -548,8 +588,14 @@ export async function listRulesForEvaluator(params: {
where: {
projectId: params.projectId,
evaluatorId: params.evaluatorId,
- evaluator: { projectId: params.projectId },
- evaluationRule: { projectId: params.projectId },
+ evaluator: {
+ projectId: params.projectId,
+ type: { not: EvalTemplateType.FACET },
+ },
+ evaluationRule: {
+ projectId: params.projectId,
+ ...visibleRuleWhere,
+ },
},
orderBy: { createdAt: "desc" },
select: {
diff --git a/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.ts b/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.ts
index c0083b84e3..3433efa358 100644
--- a/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.ts
+++ b/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.ts
@@ -5,7 +5,6 @@ import {
type FilterState,
type ModelConfig,
type EvalTemplateSourceCodeLanguage,
- type EvalTemplateType,
type EvaluatorPromptMessage,
} from "@langfuse/shared";
import { createStore, type StoreApi } from "zustand/vanilla";
@@ -21,6 +20,8 @@ import type {
import type { JudgeModel } from "@/src/features/evals/v2/judgeModel";
import type { ScoreOutputFormState } from "@/src/features/evals/v2/scoreOutputTypes";
import type { NormalizedEvaluatorDefinition } from "@/src/features/evals/v2/server/evaluators/evaluatorTypes";
+
+type EditableEvaluatorType = NormalizedEvaluatorDefinition["type"];
import { toScoreOutputFormState } from "@/src/features/evals/v2/fns/scoreOutput/toScoreOutputFormState";
import { questionsToDrafts } from "@/src/features/evals/v2/fns/evaluators/decisionModelQuestions";
import type { DecisionModelQuestionDraft } from "@/src/features/evals/v2/types/decisionModel";
@@ -91,7 +92,7 @@ function buildInitialQuestions(
}
type EvaluatorSetupStoreActions = {
- setType: (type: EvalTemplateType) => void;
+ setType: (type: EditableEvaluatorType) => void;
setQuestion: (question: DecisionModelQuestionDraft) => void;
addQuestion: (question: DecisionModelQuestionDraft) => void;
removeQuestion: (id: string) => void;
@@ -130,7 +131,7 @@ type EvaluatorSetupStoreActions = {
export type EvaluatorSetupStoreState = {
initialDefinition: NormalizedEvaluatorDefinition | undefined;
- type: EvalTemplateType;
+ type: EditableEvaluatorType;
promptMessages: EvaluatorPromptMessage[];
/** Stable client-only ids used by drag-and-drop; never persisted. */
promptMessageIds: string[];
@@ -185,7 +186,7 @@ export function createEvaluatorSetupStore({
definition: NormalizedEvaluatorDefinition;
} | null;
initialSampleFilter?: FilterState;
- initialType?: EvalTemplateType;
+ initialType?: EditableEvaluatorType;
defaultModel?: JudgeModel | null;
mode: "create" | "edit";
}): EvaluatorSetupStore {
diff --git a/web/src/features/mcp/server/evals/tools/evaluatorInput.ts b/web/src/features/mcp/server/evals/tools/evaluatorInput.ts
index e0b0ae452b..40a108858e 100644
--- a/web/src/features/mcp/server/evals/tools/evaluatorInput.ts
+++ b/web/src/features/mcp/server/evals/tools/evaluatorInput.ts
@@ -78,7 +78,7 @@ const McpEvalOutputDefinitionSchema = z.object({
export const McpEvaluatorInputBase = z.object({
name: CreateEvaluatorSchema.shape.name,
description: CreateEvaluatorSchema.shape.description.unwrap().optional(),
- type: z.enum(EvalTemplateType),
+ type: z.enum([EvalTemplateType.LLM_AS_JUDGE, EvalTemplateType.CODE]),
prompt: z.string().min(1).optional(),
modelConfig: McpEvaluatorModelConfigSchema.optional().describe(
"Optional custom model configuration. Omit to use the project default model.",
diff --git a/worker/src/features/evaluation/observationEval/createSchedulerDeps.ts b/worker/src/features/evaluation/observationEval/createSchedulerDeps.ts
index c9f3904629..18f8409cf3 100644
--- a/worker/src/features/evaluation/observationEval/createSchedulerDeps.ts
+++ b/worker/src/features/evaluation/observationEval/createSchedulerDeps.ts
@@ -106,6 +106,13 @@ export function createObservationEvalSchedulerDeps(): ObservationEvalSchedulerDe
return;
}
+ if (
+ params.evalTemplateType !== EvalTemplateType.LLM_AS_JUDGE &&
+ params.evalTemplateType !== EvalTemplateType.DECISION_MODEL
+ ) {
+ throw new Error("Evaluator type cannot use observation eval queues");
+ }
+
const queue = LLMAsJudgeExecutionQueue.getInstance({ shardingKey });
if (!queue) {
throw new Error("LLMAsJudgeExecutionQueue is not initialized");
diff --git a/worker/src/features/evaluation/observationEval/fetchObservationEvalRules.ts b/worker/src/features/evaluation/observationEval/fetchObservationEvalRules.ts
index a2a83064a8..f91dd53ae7 100644
--- a/worker/src/features/evaluation/observationEval/fetchObservationEvalRules.ts
+++ b/worker/src/features/evaluation/observationEval/fetchObservationEvalRules.ts
@@ -1,4 +1,5 @@
import {
+ EvalTemplateType,
EvalTargetObject,
JobConfigState,
coerceLegacyEmptyMetadataFilters,
@@ -13,11 +14,17 @@ import {
} from "@langfuse/shared/src/server";
import { type ObservationEvalRule } from "./types";
+const runnableEvaluatorTypes = [
+ EvalTemplateType.LLM_AS_JUDGE,
+ EvalTemplateType.CODE,
+ EvalTemplateType.DECISION_MODEL,
+];
+
/**
* Fetches the runnable observation evaluation rules for a project.
*
* This runs per ingested observation, so it stays as narrow as possible:
- * inactive rules and blocked evaluators are excluded in SQL rather than
+ * inactive rules and non-runnable evaluators are excluded in SQL rather than
* filtered in the scheduler, and evaluator versions are not joined at all —
* dispatch only needs the evaluator's identity and type, and the executor
* resolves the definition when it picks the job up.
@@ -49,9 +56,14 @@ export async function fetchObservationEvalRules(
in: [EvalTargetObject.EVENT, EvalTargetObject.EXPERIMENT],
},
status: JobConfigState.ACTIVE,
- // A rule whose every evaluator is blocked schedules nothing, so it must
- // not keep the project out of the "no rules" cache below.
- assignments: { some: { projectId, evaluator: { blockedAt: null } } },
+ // A rule without runnable assignments schedules nothing, so it must not
+ // keep the project out of the "no rules" cache below.
+ assignments: {
+ some: {
+ projectId,
+ evaluator: { blockedAt: null, type: { in: runnableEvaluatorTypes } },
+ },
+ },
},
select: {
id: true,
@@ -61,7 +73,10 @@ export async function fetchObservationEvalRules(
status: true,
targetObject: true,
assignments: {
- where: { projectId, evaluator: { blockedAt: null } },
+ where: {
+ projectId,
+ evaluator: { blockedAt: null, type: { in: runnableEvaluatorTypes } },
+ },
select: {
id: true,
evaluatorId: true,
diff --git a/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts b/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts
index c0a87bd3ba..a707256450 100644
--- a/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts
+++ b/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts
@@ -493,6 +493,8 @@ function normalizeEvalTemplate(
sourceCodeLanguage: null,
questions: template.questions,
};
+ case EvalTemplateType.FACET:
+ throw new UnrecoverableError("Facets cannot run as evaluators");
}
}
diff --git a/worker/src/features/evaluation/observationEval/scheduleObservationEvals.ts b/worker/src/features/evaluation/observationEval/scheduleObservationEvals.ts
index d9ed113ad6..032381858d 100644
--- a/worker/src/features/evaluation/observationEval/scheduleObservationEvals.ts
+++ b/worker/src/features/evaluation/observationEval/scheduleObservationEvals.ts
@@ -15,6 +15,7 @@ import {
logger,
} from "@langfuse/shared/src/server";
import {
+ EvalTemplateType,
JobExecutionStatus,
type FilterState,
type EvalExecutionMode,
@@ -308,20 +309,24 @@ function getExecutableAssignments(
rule: ObservationEvalRule,
): ScheduledObservationEvalAssignment[] {
if (!("assignments" in rule)) {
- return rule.evalTemplateId
- ? [
- {
- id: rule.id,
- evaluatorId: null,
- evaluationRuleId: null,
- evalTemplateId: rule.evalTemplateId,
- evaluatorType: rule.evalTemplate.type,
- },
- ]
- : [];
+ if (
+ !rule.evalTemplateId ||
+ rule.evalTemplate.type === EvalTemplateType.FACET
+ )
+ return [];
+ return [
+ {
+ id: rule.id,
+ evaluatorId: null,
+ evaluationRuleId: null,
+ evalTemplateId: rule.evalTemplateId,
+ evaluatorType: rule.evalTemplate.type,
+ },
+ ];
}
return rule.assignments.flatMap((assignment) => {
+ if (assignment.evaluator.type === EvalTemplateType.FACET) return [];
// Blocked evaluators are already excluded by the query; this guards the
// tenant boundary for callers that build assignments by hand.
if (assignment.evaluator.projectId !== rule.projectId) {