🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(evals): add facet type, built-in protection, and rule idle time (#18144)
Lysander Kieselcd8f35021f5f
diff --git a/packages/shared/prisma/migrations/20261001160000_add_facet_evaluator_type/migration.sql b/packages/shared/prisma/migrations/20261001160000_add_facet_evaluator_type/migration.sqlnew file mode 100644index 0000000000..c013294abe--- /dev/null+++ b/packages/shared/prisma/migrations/20261001160000_add_facet_evaluator_type/migration.sql@@ -0,0 +1,7 @@+ALTER TYPE "EvalTemplateType" ADD VALUE IF NOT EXISTS 'FACET';++ALTER TABLE "evaluators"+ ADD COLUMN IF NOT EXISTS "is_built_in" BOOLEAN NOT NULL DEFAULT FALSE;++ALTER TABLE "evaluation_rules"+ ADD COLUMN IF NOT EXISTS "idle_time" INTEGER;diff --git a/packages/shared/prisma/schema.prisma b/packages/shared/prisma/schema.prismaindex ac463e5545..36735c43ad 100644--- a/packages/shared/prisma/schema.prisma+++ b/packages/shared/prisma/schema.prisma@@ -1043,6 +1043,7 @@ model Evaluator {name Stringtype EvalTemplateType+ isBuiltIn Boolean @default(false) @map("is_built_in")description String?createdByUserId String? @map("created_by_user_id")createdByUser User? @relation("EvaluatorCreatedByUser", fields: [createdByUserId], references: [id], onDelete: SetNull)@@ -1089,6 +1090,7 @@ enum EvalTemplateType {LLM_AS_JUDGECODEDECISION_MODEL+ FACET}enum EvalTemplateSourceCodeLanguage {@@ -1167,6 +1169,7 @@ model EvaluationRule {filter Jsonsampling Decimaldelay Int+ idleTime Int? @map("idle_time") // millisecondstimeScope String[] @default(["NEW"]) @map("time_scope")assignments EvaluationRuleEvaluatorAssignment[]diff --git a/packages/shared/src/features/evals/types.ts b/packages/shared/src/features/evals/types.tsindex d8b572d9ea..47d0651b6d 100644--- a/packages/shared/src/features/evals/types.ts+++ b/packages/shared/src/features/evals/types.ts@@ -16,6 +16,7 @@ export const EvalTemplateTypeEnum = {LLM_AS_JUDGE: "LLM_AS_JUDGE",CODE: "CODE",DECISION_MODEL: "DECISION_MODEL",+ FACET: "FACET",} as const satisfies Record<EvalTemplateType, EvalTemplateType>;export const EvalTemplateSourceCodeLanguageEnum = {diff --git a/web/src/__tests__/server/evals-trpc.servertest.ts b/web/src/__tests__/server/evals-trpc.servertest.tsindex 19623ac893..635191e08d 100644--- a/web/src/__tests__/server/evals-trpc.servertest.ts+++ b/web/src/__tests__/server/evals-trpc.servertest.ts@@ -114,6 +114,76 @@ describe("legacy evaluator compatibility service", () => {expect(config?.evalTemplate?.name).toBe("Legacy score");});+ it("keeps facet rules out of legacy evaluator reads and writes", async () => {+ const { project, service } = await prepare();+ const rule = await createLegacyRule(project.id);+ await prisma.evaluator.updateMany({+ where: { projectId: project.id },+ data: { type: EvalTemplateType.FACET, isBuiltIn: true },+ });++ await expect(service.counts(project.id)).resolves.toEqual({+ configCount: 0,+ configActiveCount: 0,+ templateCount: 0,+ legacyConfigCount: 0,+ });+ await expect(+ service.listConfigs({ projectId: project.id }),+ ).resolves.toEqual({+ configs: [],+ totalCount: 0,+ });+ await expect(service.getConfig(project.id, rule.id)).resolves.toBeNull();+ await expect(+ service.listConfigsByTemplateName(project.id, "Legacy score"),+ ).resolves.toEqual([]);+ await expect(+ service.updateConfig({+ projectId: project.id,+ ruleId: rule.id,+ data: { status: JobConfigState.INACTIVE },+ }),+ ).rejects.toThrow("Evaluation rule not found");+ await expect(service.deleteConfig(project.id, rule.id)).resolves.toBe(+ false,+ );+ });++ it("rejects activation when a built-in evaluator remains blocked", async () => {+ const { project, service } = await prepare();+ const rule = await createLegacyRule(project.id);+ await prisma.evaluationRule.update({+ where: { id: rule.id },+ data: { status: JobConfigState.INACTIVE },+ });+ await prisma.evaluator.updateMany({+ where: { projectId: project.id },+ data: { isBuiltIn: true, blockedAt: new Date() },+ });++ await expect(+ service.updateConfig({+ projectId: project.id,+ ruleId: rule.id,+ data: { status: JobConfigState.ACTIVE },+ }),+ ).rejects.toThrow("Blocked built-in evaluators cannot be activated");+ await expect(+ service.setConfigStatuses({+ projectId: project.id,+ ruleIds: [rule.id],+ status: JobConfigState.ACTIVE,+ }),+ ).rejects.toThrow("Blocked built-in evaluators cannot be activated");+ await expect(+ prisma.evaluationRule.findUnique({+ where: { id: rule.id },+ select: { status: true },+ }),+ ).resolves.toEqual({ status: JobConfigState.INACTIVE });+ });+it("projects an unassigned legacy-target rule for its read-only view", async () => {const { project, service } = await prepare();const rule = await createLegacyRule(project.id);diff --git a/web/src/__tests__/server/evaluation-rule-v2-repository.servertest.ts b/web/src/__tests__/server/evaluation-rule-v2-repository.servertest.tsindex 13721e1928..a0d647d2f2 100644--- a/web/src/__tests__/server/evaluation-rule-v2-repository.servertest.ts+++ b/web/src/__tests__/server/evaluation-rule-v2-repository.servertest.ts@@ -1,5 +1,9 @@import { randomUUID } from "node:crypto";-import { EvalTargetObject, type FilterState } from "@langfuse/shared";+import {+ EvalTargetObject,+ EvalTemplateType,+ type FilterState,+} from "@langfuse/shared";import { Prisma, prisma } from "@langfuse/shared/src/db";import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";@@ -99,6 +103,42 @@ async function createRule({}describe("evaluation rule v2 repository", () => {+ it("excludes facet rules from user-facing reads and mutations", async () => {+ const evaluator = await createEvaluator();+ await prisma.evaluator.update({+ where: { id: evaluator.id },+ data: { type: EvalTemplateType.FACET, isBuiltIn: true },+ });+ const rule = await createRule({ evaluatorId: evaluator.id });++ await expect(+ ruleRepository.listRules({+ prisma,+ input: { projectId, page: 1, limit: 50 },+ }),+ ).resolves.toEqual({ rules: [], totalItems: 0 });+ await expect(+ ruleRepository.findRule({ prisma, projectId, ruleId: rule.id }),+ ).resolves.toBeNull();+ await expect(+ ruleRepository.listSelectedRuleIds({+ prisma,+ input: { projectId, ruleIds: [rule.id] },+ }),+ ).resolves.toEqual([]);+ await expect(+ ruleRepository.setRuleStatus({+ prisma,+ projectId,+ ruleIds: [rule.id],+ enabled: false,+ }),+ ).resolves.toEqual({ count: 0 });+ await expect(+ ruleRepository.deleteRule({ prisma, projectId, ruleId: rule.id }),+ ).resolves.toBe(false);+ });+describe("listRules", () => {it("returns an empty list", async () => {await expect(diff --git a/web/src/__tests__/server/unit/batch-actions-comment-filter.servertest.ts b/web/src/__tests__/server/unit/batch-actions-comment-filter.servertest.tsindex ad04d2dc3b..a26560e397 100644--- a/web/src/__tests__/server/unit/batch-actions-comment-filter.servertest.ts+++ b/web/src/__tests__/server/unit/batch-actions-comment-filter.servertest.ts@@ -311,6 +311,7 @@ describe("batched evaluation version selection", () => {where: {id: { in: [evaluatorId] },projectId,+ type: { not: "FACET" },assignments: {none: {evaluationRule: {diff --git a/web/src/features/evals/server/legacyCompatibilityService.ts b/web/src/features/evals/server/legacyCompatibilityService.tsindex dbbc97336a..f3e67e96a5 100644--- a/web/src/features/evals/server/legacyCompatibilityService.ts+++ b/web/src/features/evals/server/legacyCompatibilityService.ts@@ -61,6 +61,39 @@ const LEGACY_TARGET_OBJECTS = [EvalTargetObject.EXPERIMENT,];+const visibleRuleWhere = {+ assignments: { none: { evaluator: { type: EvalTemplateType.FACET } } },+} satisfies Prisma.EvaluationRuleWhereInput;++async function assertNoBlockedBuiltInEvaluators(+ tx: TransactionClient,+ projectId: string,+ ruleIds: string[],+) {+ const blockedRule = await tx.evaluationRule.findFirst({+ where: {+ projectId,+ id: { in: ruleIds },+ AND: [+ visibleRuleWhere,+ {+ assignments: {+ some: {+ evaluator: { isBuiltIn: true, blockedAt: { not: null } },+ },+ },+ },+ ],+ },+ select: { id: true },+ });+ if (blockedRule) {+ throw new InvalidRequestError(+ "Blocked built-in evaluators cannot be activated",+ );+ }+}+const latestVersion = {orderBy: { version: "desc" as const },take: 1,@@ -250,6 +283,7 @@ function definitionFromEvaluator(): EvaluatorDefinition | null {const version = evaluator.versions[0];if (!version) return null;+ if (evaluator.type === EvalTemplateType.FACET) return null;if (evaluator.type === EvalTemplateType.CODE) {if (!version.sourceCode || !version.sourceCodeLanguage) return null;return {@@ -389,7 +423,10 @@ async function findReusableEvaluatorId(params: {}const version = await tx.evaluatorVersion.findFirst({- where: { id: templateId, evaluator: { projectId } },+ where: {+ id: templateId,+ evaluator: { projectId, type: { not: EvalTemplateType.FACET } },+ },include: { evaluator: { include: { versions: latestVersion } } },});if (!version) return null;@@ -410,6 +447,7 @@ function isRunnableTemplate(template: {type: EvalTemplateType;sourceCodeLanguage: EvalTemplateSourceCodeLanguage | null;}) {+ if (template.type === EvalTemplateType.FACET) return false;if (template.type !== EvalTemplateType.CODE) return true;return (@@ -497,7 +535,12 @@ async function clearEvaluatorBlock(params: {if (!assignment) return;await params.tx.evaluator.updateMany({- where: { id: assignment.evaluatorId, projectId: params.projectId },+ where: {+ id: assignment.evaluatorId,+ projectId: params.projectId,+ isBuiltIn: false,+ type: { not: EvalTemplateType.FACET },+ },data: resetEvalConfigBlockFields,});}@@ -523,10 +566,13 @@ async function applyScoreNameChange(params: {},});if (!evaluator || evaluator.name === scoreName) return;+ if (evaluator.isBuiltIn || evaluator.type === EvalTemplateType.FACET) {+ throw new InvalidRequestError("Built-in evaluators cannot be edited");+ }if (evaluator._count.assignments <= 1) {await tx.evaluator.update({- where: { id: evaluatorId, projectId },+ where: { id: evaluatorId, projectId, isBuiltIn: false },data: { name: scoreName },});return;@@ -611,6 +657,7 @@ const legacyConfigIdsQuery = (params: {) a ON a."evaluation_rule_id" = r."id"JOIN "evaluators" e ON e."id" = a."evaluator_id"WHERE r."project_id" = ${params.projectId}+ AND e."type"::text <> ${EvalTemplateType.FACET}) jcWHERE TRUE${params.targetCondition}@@ -647,14 +694,23 @@ export class LegacyEvalCompatibilityService {async counts(projectId: string) {const [configCount, configActiveCount, templateCount, legacyConfigCount] =await Promise.all([- this.prisma.evaluationRule.count({ where: { projectId } }),this.prisma.evaluationRule.count({- where: { projectId, status: JobConfigState.ACTIVE },+ where: { projectId, ...visibleRuleWhere },}),- this.prisma.evaluator.count({ where: { projectId } }),this.prisma.evaluationRule.count({where: {projectId,+ ...visibleRuleWhere,+ status: JobConfigState.ACTIVE,+ },+ }),+ this.prisma.evaluator.count({+ where: { projectId, type: { not: EvalTemplateType.FACET } },+ }),+ this.prisma.evaluationRule.count({+ where: {+ projectId,+ ...visibleRuleWhere,targetObject: {in: [EvalTargetObject.TRACE, EvalTargetObject.DATASET],},@@ -757,7 +813,7 @@ export class LegacyEvalCompatibilityService {async getConfig(projectId: string, ruleId: string) {const rule = await this.prisma.evaluationRule.findFirst({- where: { id: ruleId, projectId },+ where: { id: ruleId, projectId, ...visibleRuleWhere },include: ruleInclude,});if (!rule || rule.assignments.length > 1) return null;@@ -811,7 +867,7 @@ export class LegacyEvalCompatibilityService {options: { collapseManagedCopies?: boolean } = {},) {const evaluators = await this.prisma.evaluator.findMany({- where: { projectId },+ where: { projectId, type: { not: EvalTemplateType.FACET } },include: { versions: latestVersion },orderBy: [{ name: "asc" }, { createdAt: "asc" }],});@@ -835,6 +891,7 @@ export class LegacyEvalCompatibilityService {const evaluators = await this.prisma.evaluator.findMany({where: {projectId: params.projectId,+ type: { not: EvalTemplateType.FACET },...(search? { name: { contains: search, mode: "insensitive" as const } }: {}),@@ -947,7 +1004,10 @@ export class LegacyEvalCompatibilityService {return isRunnableTemplate(legacyTemplate) ? legacyTemplate : null;}const version = await this.prisma.evaluatorVersion.findFirst({- where: { id: templateId, evaluator: { projectId } },+ where: {+ id: templateId,+ evaluator: { projectId, type: { not: EvalTemplateType.FACET } },+ },include: { evaluator: true },});if (!version) return null;@@ -979,7 +1039,10 @@ export class LegacyEvalCompatibilityService {const version = await this.prisma.evaluatorVersion.findFirst({where: {id: params.templateId,- evaluator: { projectId: params.projectId },+ evaluator: {+ projectId: params.projectId,+ type: { not: EvalTemplateType.FACET },+ },},include: { evaluator: { include: { versions: latestVersion } } },});@@ -1048,6 +1111,7 @@ export class LegacyEvalCompatibilityService {where: {id: params.reuseEvaluatorFromRuleId,projectId: params.projectId,+ ...visibleRuleWhere,targetObject: { in: LEGACY_TARGET_OBJECTS },},select: {@@ -1154,7 +1218,11 @@ export class LegacyEvalCompatibilityService {const source = await tx.evaluatorVersion.findFirst({where: {id: params.intent.sourceTemplateId,- evaluator: { projectId: params.projectId },+ evaluator: {+ projectId: params.projectId,+ isBuiltIn: false,+ type: { not: EvalTemplateType.FACET },+ },},select: { evaluatorId: true },});@@ -1168,6 +1236,9 @@ export class LegacyEvalCompatibilityService {where: { id: source.evaluatorId, projectId: params.projectId },include: { versions: latestVersion },});+ if (evaluator.isBuiltIn) {+ throw new InvalidRequestError("Built-in evaluators cannot be edited");+ }if (evaluator.name !== params.name ||evaluator.type !== params.definition.type@@ -1309,7 +1380,7 @@ export class LegacyEvalCompatibilityService {async listTemplateVersions(projectId: string, name: string) {const evaluators = await this.prisma.evaluator.findMany({- where: { projectId, name },+ where: { projectId, name, type: { not: EvalTemplateType.FACET } },include: { versions: { orderBy: { version: "desc" } } },});return evaluators.flatMap((evaluator) =>@@ -1327,7 +1398,10 @@ export class LegacyEvalCompatibilityService {const rules = await this.prisma.evaluationRule.findMany({where: {projectId,- assignments: { some: { evaluator: { name } } },+ AND: [+ visibleRuleWhere,+ { assignments: { some: { evaluator: { name } } } },+ ],},include: ruleInclude,});@@ -1354,6 +1428,7 @@ export class LegacyEvalCompatibilityService {where: {id: params.ruleId,projectId: params.projectId,+ ...visibleRuleWhere,targetObject: { in: LEGACY_TARGET_OBJECTS },},include: { assignments: true },@@ -1370,6 +1445,7 @@ export class LegacyEvalCompatibilityService {});}if (params.data.status === JobConfigState.ACTIVE) {+ await assertNoBlockedBuiltInEvaluators(tx, params.projectId, [rule.id]);// Blocking moved to the evaluator, so activating the rule alone would// leave the evaluator paused and the worker would keep skipping it.await clearEvaluatorBlock({@@ -1432,10 +1508,18 @@ export class LegacyEvalCompatibilityService {if (params.ruleIds.length === 0) return 0;return this.prisma.$transaction(async (tx) => {+ if (params.status === JobConfigState.ACTIVE) {+ await assertNoBlockedBuiltInEvaluators(+ tx,+ params.projectId,+ params.ruleIds,+ );+ }const { count } = await tx.evaluationRule.updateMany({where: {id: { in: params.ruleIds },projectId: params.projectId,+ ...visibleRuleWhere,targetObject: { in: LEGACY_TARGET_OBJECTS },},data: { status: params.status },@@ -1447,6 +1531,7 @@ export class LegacyEvalCompatibilityService {where: {projectId: params.projectId,evaluationRuleId: { in: params.ruleIds },+ evaluationRule: visibleRuleWhere,},select: { evaluatorId: true },},@@ -1455,6 +1540,8 @@ export class LegacyEvalCompatibilityService {where: {projectId: params.projectId,id: { in: assignments.map(({ evaluatorId }) => evaluatorId) },+ isBuiltIn: false,+ type: { not: EvalTemplateType.FACET },},data: resetEvalConfigBlockFields,});@@ -1469,6 +1556,7 @@ export class LegacyEvalCompatibilityService {where: {id: ruleId,projectId,+ ...visibleRuleWhere,targetObject: { in: LEGACY_TARGET_OBJECTS },},select: { id: true },@@ -1493,7 +1581,10 @@ export class LegacyEvalCompatibilityService {return [];}const version = await this.prisma.evaluatorVersion.findFirst({- where: { id: templateId, evaluator: { projectId } },+ where: {+ id: templateId,+ evaluator: { projectId, type: { not: EvalTemplateType.FACET } },+ },select: { evaluatorId: true },});if (!version) return [];@@ -1517,7 +1608,14 @@ export class LegacyEvalCompatibilityService {return this.prisma.$transaction(async (tx) => {const version = await tx.evaluatorVersion.findFirst({- where: { id: templateId, evaluator: { projectId } },+ where: {+ id: templateId,+ evaluator: {+ projectId,+ isBuiltIn: false,+ type: { not: EvalTemplateType.FACET },+ },+ },select: { evaluatorId: true },});if (!version) throw new LangfuseNotFoundError("Evaluator not found");@@ -1543,7 +1641,7 @@ export class LegacyEvalCompatibilityService {where: { evaluatorId: version.evaluatorId },});await tx.evaluator.delete({- where: { id: version.evaluatorId, projectId },+ where: { id: version.evaluatorId, projectId, isBuiltIn: false },});return versions;});diff --git a/web/src/features/evals/v2/AGENTS.md b/web/src/features/evals/v2/AGENTS.mdindex 77a0cdfb34..aeafd59a6b 100644--- a/web/src/features/evals/v2/AGENTS.md+++ b/web/src/features/evals/v2/AGENTS.md@@ -12,6 +12,8 @@ The new data model is- Rule (basically the old `job_configuration`)- Rule assignments (association table handling the n:m relationship)+FACET evaluators are internal and must be excluded from user-facing evaluator and rule queries.+Traces captured during the eval executions in the past only captured `job_configuration_id`.Only new runs capture `evaluator_id` and `evaluation_rule_id`.diff --git a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryMethodBadge/EvaluatorGalleryMethodBadge.tsx b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryMethodBadge/EvaluatorGalleryMethodBadge.tsxindex 129d111d87..b43e8fe81b 100644--- a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryMethodBadge/EvaluatorGalleryMethodBadge.tsx+++ b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryMethodBadge/EvaluatorGalleryMethodBadge.tsx@@ -1,4 +1,4 @@-import { Code2, Scale, Sparkles } from "lucide-react";+import { Code2, Scale, Sparkles, Tags } from "lucide-react";import { EvalTemplateTypeEnum, type EvalTemplateType } from "@langfuse/shared";const METHOD_BADGES = {@@ -11,6 +11,7 @@ const METHOD_BADGES = {Icon: Scale,label: "Decision model",},+ [EvalTemplateTypeEnum.FACET]: { Icon: Tags, label: "Facet" },} as const satisfies Record<EvalTemplateType, { Icon: unknown; label: string }>;export function EvaluatorGalleryMethodBadge({diff --git a/web/src/features/evals/v2/components/EvaluatorTestPanel/components/TestSection/components/TestSectionContainer/TestSectionContainer.tsx b/web/src/features/evals/v2/components/EvaluatorTestPanel/components/TestSection/components/TestSectionContainer/TestSectionContainer.tsxindex b1ab135cb7..34b6eb61c9 100644--- a/web/src/features/evals/v2/components/EvaluatorTestPanel/components/TestSection/components/TestSectionContainer/TestSectionContainer.tsx+++ b/web/src/features/evals/v2/components/EvaluatorTestPanel/components/TestSection/components/TestSectionContainer/TestSectionContainer.tsx@@ -12,7 +12,10 @@ import { TestRunCard } from "@/src/features/evals/v2/components/EvaluatorTestPanimport { toTestResultPanelState } from "@/src/features/evals/v2/fns/evaluatorTesting/toTestResultPanelState";import type { EvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";-const RESULT_TITLES: Record<EvalTemplateType, TestResultPanelTitle> = {+const RESULT_TITLES: Record<+ Exclude<EvalTemplateType, "FACET">,+ TestResultPanelTitle+> = {LLM_AS_JUDGE: "LLM Output",CODE: "Code Output",DECISION_MODEL: "Decision Model Output",diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DefinitionStepContainer/DefinitionStepContainer.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DefinitionStepContainer/DefinitionStepContainer.tsxindex c1e0532d2a..39e2771ee8 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DefinitionStepContainer/DefinitionStepContainer.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DefinitionStepContainer/DefinitionStepContainer.tsx@@ -53,6 +53,7 @@ export function DefinitionStepContainer({})),);const changeType = (type: EvalTemplateType) => {+ if (type === "FACET") return;const previousEvaluatorType = store.getState().type;state.actions.setType(type);if (type !== previousEvaluatorType) {diff --git a/web/src/features/evals/v2/fns/evaluators/evaluatorTypeLabel.ts b/web/src/features/evals/v2/fns/evaluators/evaluatorTypeLabel.tsindex 4deabacb3d..7a3ac04bb0 100644--- a/web/src/features/evals/v2/fns/evaluators/evaluatorTypeLabel.ts+++ b/web/src/features/evals/v2/fns/evaluators/evaluatorTypeLabel.ts@@ -9,5 +9,7 @@ export function evaluatorTypeLabel(type: EvalTemplateType): string {return "Decision model (experimental)";case EvalTemplateTypeEnum.LLM_AS_JUDGE:return "LLM as a judge";+ case EvalTemplateTypeEnum.FACET:+ return "Facet";}}diff --git a/web/src/features/evals/v2/pages/EvaluatorDetailPage.tsx b/web/src/features/evals/v2/pages/EvaluatorDetailPage.tsxindex 51a9fa03c0..e272af5909 100644--- a/web/src/features/evals/v2/pages/EvaluatorDetailPage.tsx+++ b/web/src/features/evals/v2/pages/EvaluatorDetailPage.tsx@@ -39,6 +39,8 @@ function toSetupDefinition(sourceCode: latest.sourceCode ?? "",sourceCodeLanguage: latest.sourceCodeLanguage ?? "TYPESCRIPT",};+ case "FACET":+ throw new Error("Facets are not shown as evaluators");}}@@ -54,7 +56,7 @@ export default function EvaluatorDetailPage() {if (evaluator.isPending) {return <EvaluatorSetupLoadingPage mode="edit" projectId={projectId} />;}- if (!evaluator.data?.versions[0]) {+ if (!evaluator.data?.versions[0] || evaluator.data.type === "FACET") {return <div className="p-6">Evaluator not found</div>;}diff --git a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsxindex 7c0bb5e926..a81fd9699c 100644--- a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx+++ b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx@@ -77,7 +77,7 @@ type InitialEvaluator = {id: string;name: string;description: string | null;- type: EvalTemplateType;+ type: Exclude<EvalTemplateType, "FACET">;definition: NormalizedEvaluatorDefinition;blockedAt: Date | null;blockReason: EvaluatorBlockReason | null;@@ -103,6 +103,9 @@ export function applyEvaluatorSuggestion(export function getEvaluatorVersionDefinition(version: EvaluatorVersion,): NormalizedEvaluatorDefinition {+ if (version.type === "FACET") {+ throw new Error("Facets cannot be edited as evaluators");+ }if (version.type === "CODE") {return {type: version.type,@@ -167,7 +170,7 @@ export function EvaluatorSetupPage(mode: "create";projectId: string;initialDraft: EvaluatorSetupDraft | null;- initialType: EvalTemplateType;+ initialType: Exclude<EvalTemplateType, "FACET">;creationSource: EvaluatorCreationSource;}| {diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts b/web/src/features/evals/v2/server/evaluators/evaluatorRepository.tsindex 67f016ece9..60bc265154 100644--- a/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts+++ b/web/src/features/evals/v2/server/evaluators/evaluatorRepository.ts@@ -45,6 +45,10 @@ const eventEvaluatorFilterColumnIds = new Set(eventsEvalFilterColumns.map((column) => column.id),);+const visibleEvaluatorWhere = {+ type: { not: EvalTemplateType.FACET },+} as const satisfies Prisma.EvaluatorWhereInput;+const publicEvaluationRuleAssignments = (projectId: string) =>({where: { projectId },@@ -58,6 +62,7 @@ const publicEvaluationRuleAssignments = (projectId: string) =>}) satisfies Prisma.EvaluationRuleEvaluatorAssignmentFindManyArgs;export const batchEligibleEvaluatorWhere = {+ ...visibleEvaluatorWhere,// Trace/dataset assignments carry rule-specific mappings that an// observation batch cannot resolve when it addresses the evaluator alone.assignments: {@@ -188,6 +193,7 @@ async function evaluatorIdsMatchingModelFilters(params: {LEFT JOIN default_llm_models AS default_modelON default_model.project_id = evaluator.project_idWHERE evaluator.project_id = ${params.projectId}+ AND evaluator.type <> 'FACET'AND ${Prisma.join(predicates, " AND ")}`,);@@ -272,6 +278,7 @@ async function evaluatorWhere(params: {return {projectId: params.projectId,+ ...visibleEvaluatorWhere,...(modelEvaluatorIds ? { id: { in: modelEvaluatorIds } } : {}),...(params.search? { name: { contains: params.search, mode: "insensitive" as const } }@@ -352,6 +359,7 @@ function getEffectiveModel(case EvalTemplateType.DECISION_MODEL:return evaluator.versions[0]?.model ?? null;case EvalTemplateType.CODE:+ case EvalTemplateType.FACET:return null;}}@@ -365,8 +373,10 @@ export async function listEvaluatorsCursor(params: {types?: EvalTemplateType[];}) {const baseWhere: Prisma.EvaluatorWhereInput = {- ...(await evaluatorWhere(params)),- ...(params.types ? { type: { in: params.types } } : {}),+ AND: [+ await evaluatorWhere(params),+ ...(params.types ? [{ type: { in: params.types } }] : []),+ ],};const where: Prisma.EvaluatorWhereInput = params.cursor? {@@ -419,7 +429,7 @@ export async function listEvaluatorFilterOptions(params: {}) {const [evaluators, defaultModel] = await Promise.all([params.prisma.evaluator.findMany({- where: { projectId: params.projectId },+ where: { projectId: params.projectId, ...visibleEvaluatorWhere },select: {name: true,type: true,@@ -477,7 +487,11 @@ export function countProjectEvaluators(params: {evaluatorIds: string[];}) {return params.prisma.evaluator.count({- where: { projectId: params.projectId, id: { in: params.evaluatorIds } },+ where: {+ projectId: params.projectId,+ id: { in: params.evaluatorIds },+ ...visibleEvaluatorWhere,+ },});}@@ -491,6 +505,7 @@ export async function listEvaluatorOptions(params: {const evaluators = await params.prisma.evaluator.findMany({where: {projectId: params.projectId,+ ...visibleEvaluatorWhere,...(params.search? { name: { contains: params.search, mode: "insensitive" } }: {}),@@ -530,7 +545,11 @@ export function findEvaluator(params: {evaluatorId: string;}) {return params.prisma.evaluator.findFirst({- where: { id: params.evaluatorId, projectId: params.projectId },+ where: {+ id: params.evaluatorId,+ projectId: params.projectId,+ ...visibleEvaluatorWhere,+ },include: {versions: latestVersionWithCreator,createdByUser,@@ -581,6 +600,7 @@ export function findEvaluatorsByIds(params: {where: {id: { in: params.evaluatorIds },projectId: params.projectId,+ ...visibleEvaluatorWhere,},include: {versions: latestVersion,@@ -598,7 +618,7 @@ export async function listEvaluatorVersions(params: {const versions = await params.prisma.evaluatorVersion.findMany({where: {evaluatorId: params.evaluatorId,- evaluator: { projectId: params.projectId },+ evaluator: { projectId: params.projectId, ...visibleEvaluatorWhere },...(params.cursor ? { version: { lt: params.cursor } } : {}),},orderBy: { version: "desc" },@@ -623,7 +643,11 @@ export function findEvaluatorsByName(params: {name: string;}) {return params.prisma.evaluator.findMany({- where: { projectId: params.projectId, name: params.name },+ where: {+ projectId: params.projectId,+ name: params.name,+ ...visibleEvaluatorWhere,+ },include: { versions: latestVersion },take: 2,});@@ -675,7 +699,12 @@ export function blockEvaluator(params: {message: string;}) {return params.tx.evaluator.update({- where: { id: params.evaluatorId, projectId: params.projectId },+ where: {+ id: params.evaluatorId,+ projectId: params.projectId,+ isBuiltIn: false,+ ...visibleEvaluatorWhere,+ },data: {blockedAt: new Date(),blockReason: params.reason,@@ -690,7 +719,12 @@ export function unblockEvaluator(params: {evaluatorId: string;}) {return params.tx.evaluator.update({- where: { id: params.evaluatorId, projectId: params.projectId },+ where: {+ id: params.evaluatorId,+ projectId: params.projectId,+ isBuiltIn: false,+ ...visibleEvaluatorWhere,+ },data: {blockedAt: null,blockReason: null,@@ -707,7 +741,12 @@ export function updateEvaluatorMetadata(params: {description?: string | null;}) {return params.tx.evaluator.update({- where: { id: params.evaluatorId, projectId: params.projectId },+ where: {+ id: params.evaluatorId,+ projectId: params.projectId,+ isBuiltIn: false,+ ...visibleEvaluatorWhere,+ },data: {...(params.name === undefined ? {} : { name: params.name }),...(params.description === undefined@@ -757,7 +796,12 @@ export async function deleteEvaluator(params: {select: { evaluationRuleId: true },});const result = await params.prisma.evaluator.deleteMany({- where: { id: params.evaluatorId, projectId: params.projectId },+ where: {+ id: params.evaluatorId,+ projectId: params.projectId,+ isBuiltIn: false,+ ...visibleEvaluatorWhere,+ },});if (result.count > 0) {await setRuleStatus({diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorService.ts b/web/src/features/evals/v2/server/evaluators/evaluatorService.tsindex cee41c01ef..872a2aa68e 100644--- a/web/src/features/evals/v2/server/evaluators/evaluatorService.ts+++ b/web/src/features/evals/v2/server/evaluators/evaluatorService.ts@@ -483,6 +483,7 @@ export class EvaluatorService {evaluatorId,});if (!evaluator) throw new LangfuseNotFoundError("Evaluator not found");+ assertEditableEvaluator(evaluator);if (!evaluator.blockedAt)return normalizeEvaluatorPromptMessages(evaluator);@@ -631,7 +632,11 @@ export class EvaluatorService {if (definition) {if (evaluatorId) {const evaluator = await this.prisma.evaluator.findFirst({- where: { id: evaluatorId, projectId: params.projectId },+ where: {+ id: evaluatorId,+ projectId: params.projectId,+ type: { not: EvalTemplateType.FACET },+ },select: { id: true },});// The setup editor pre-generates a UUID so a test run can be attributed@@ -729,10 +734,23 @@ async function deleteEvaluator(params: {projectId: string;evaluatorId: string;}) {+ const evaluator = await repository.findEvaluator({+ prisma: params.prisma,+ projectId: params.projectId,+ evaluatorId: params.evaluatorId,+ });+ if (!evaluator) throw new LangfuseNotFoundError("Evaluator not found");+ assertEditableEvaluator(evaluator);const deleted = await repository.deleteEvaluator(params);if (!deleted) throw new LangfuseNotFoundError("Evaluator not found");}+function assertEditableEvaluator(evaluator: { isBuiltIn: boolean }) {+ if (evaluator.isBuiltIn) {+ throw new InvalidRequestError("Built-in evaluators cannot be edited");+ }+}+async function patchEvaluator(params: {tx: Prisma.TransactionClient;input: Omit<PatchEvaluatorInput, "definition"> & {@@ -748,6 +766,7 @@ async function patchEvaluator(params: {evaluatorId: input.evaluatorId,});if (!current) throw new LangfuseNotFoundError("Evaluator not found");+ assertEditableEvaluator(current);if (input.definition && current.type !== input.definition.type) {throw new LangfuseConflictError("Evaluator type cannot be changed");}@@ -811,6 +830,7 @@ async function updateEvaluator(params: {evaluatorId: input.evaluatorId,});if (!current) throw new LangfuseNotFoundError("Evaluator not found");+ assertEditableEvaluator(current);if (current.type !== input.definition.type) {throw new LangfuseConflictError("Evaluator type cannot be changed");}diff --git a/web/src/features/evals/v2/server/rules/ruleRepository.ts b/web/src/features/evals/v2/server/rules/ruleRepository.tsindex ac4e5c6804..8acd59cff6 100644--- a/web/src/features/evals/v2/server/rules/ruleRepository.ts+++ b/web/src/features/evals/v2/server/rules/ruleRepository.ts@@ -6,6 +6,7 @@ import {import {coerceLegacyEmptyMetadataFilters,EvalTargetObject,+ EvalTemplateType,type FilterState,type ObservationVariableMapping,} from "@langfuse/shared";@@ -30,6 +31,10 @@ export type RulePrisma = PrismaClient | Prisma.TransactionClient;const MAX_REUSABLE_FILTER_CANDIDATES = 20;+const visibleRuleWhere = {+ assignments: { none: { evaluator: { type: EvalTemplateType.FACET } } },+} satisfies Prisma.EvaluationRuleWhereInput;+const latestVersion = {orderBy: { version: "desc" as const },take: 1,@@ -110,6 +115,7 @@ function ruleWhere(params: {return {projectId: params.projectId,+ ...visibleRuleWhere,...(params.search? { name: { contains: params.search, mode: "insensitive" as const } }: {}),@@ -200,6 +206,7 @@ export async function listReusableFilterCandidates(params: {const rules = await params.prisma.evaluationRule.findMany({where: {projectId: params.projectId,+ ...visibleRuleWhere,targetObject: {in: [EvalTargetObject.EVENT, EvalTargetObject.EXPERIMENT],},@@ -232,7 +239,7 @@ export async function listRuleFilterOptions(params: {}) {const [rules, upgradeRequiredRule] = await Promise.all([params.prisma.evaluationRule.findMany({- where: { projectId: params.projectId },+ where: { projectId: params.projectId, ...visibleRuleWhere },select: {name: true,createdByUser: { select: { id: true, name: true, email: true } },@@ -241,6 +248,7 @@ export async function listRuleFilterOptions(params: {params.prisma.evaluationRule.findFirst({where: {projectId: params.projectId,+ ...visibleRuleWhere,...upgradeRequiredRuleCondition,},select: { id: true },@@ -267,7 +275,11 @@ export function findRule(params: {ruleId: string;}) {return params.prisma.evaluationRule.findFirst({- where: { id: params.ruleId, projectId: params.projectId },+ where: {+ id: params.ruleId,+ projectId: params.projectId,+ ...visibleRuleWhere,+ },include: ruleInclude,});}@@ -281,6 +293,7 @@ export async function findActiveRuleWithMatchingFilterAndSampling(params: {const rules = await params.prisma.evaluationRule.findMany({where: {projectId: params.projectId,+ ...visibleRuleWhere,status: JobConfigState.ACTIVE,targetObject: EvalTargetObject.EVENT,sampling: params.sampling,@@ -340,7 +353,11 @@ export function updateRule(params: {filter?: Prisma.InputJsonValue;}) {return params.prisma.evaluationRule.update({- where: { id: params.input.ruleId, projectId: params.input.projectId },+ where: {+ id: params.input.ruleId,+ projectId: params.input.projectId,+ ...visibleRuleWhere,+ },data: {...(params.targetObject === undefined? {}@@ -376,6 +393,7 @@ export function setRuleStatus(params: {return params.prisma.evaluationRule.updateMany({where: {projectId: params.projectId,+ ...visibleRuleWhere,...(params.ruleIds === undefined ? {} : { id: { in: params.ruleIds } }),...(params.unassignedOnly ? { assignments: { none: {} } } : {}),...(params.sampling === undefined ? { status: { not: status } } : {}),@@ -393,7 +411,11 @@ export async function deleteRule(params: {ruleId: string;}) {const result = await params.prisma.evaluationRule.deleteMany({- where: { id: params.ruleId, projectId: params.projectId },+ where: {+ id: params.ruleId,+ projectId: params.projectId,+ ...visibleRuleWhere,+ },});if (result.count === 0) return false;@@ -410,13 +432,28 @@ export async function deleteRules(params: {projectId: string;ruleIds: string[];}) {+ const visibleIds = await params.prisma.evaluationRule.findMany({+ where: {+ projectId: params.projectId,+ id: { in: params.ruleIds },+ ...visibleRuleWhere,+ },+ select: { id: true },+ });+ const ruleIds = visibleIds.map(({ id }) => id);+ if (ruleIds.length === 0) return 0;+const result = await params.prisma.evaluationRule.deleteMany({- where: { projectId: params.projectId, id: { in: params.ruleIds } },+ where: {+ projectId: params.projectId,+ id: { in: ruleIds },+ ...visibleRuleWhere,+ },});await params.prisma.jobExecution.deleteMany({where: {projectId: params.projectId,- jobConfigurationId: { in: params.ruleIds },+ jobConfigurationId: { in: ruleIds },},});return result.count;@@ -431,6 +468,7 @@ export async function listSelectedRuleIds(params: {where: {projectId: params.input.projectId,id: { in: params.input.ruleIds },+ ...visibleRuleWhere,},select: { id: true },});@@ -529,6 +567,8 @@ export async function countRulesForEvaluators(params: {where: {projectId: params.projectId,evaluatorId: { in: params.evaluatorIds },+ evaluator: { type: { not: EvalTemplateType.FACET } },+ evaluationRule: visibleRuleWhere,},_count: { _all: true },});@@ -548,8 +588,14 @@ export async function listRulesForEvaluator(params: {where: {projectId: params.projectId,evaluatorId: params.evaluatorId,- evaluator: { projectId: params.projectId },- evaluationRule: { projectId: params.projectId },+ evaluator: {+ projectId: params.projectId,+ type: { not: EvalTemplateType.FACET },+ },+ evaluationRule: {+ projectId: params.projectId,+ ...visibleRuleWhere,+ },},orderBy: { createdAt: "desc" },select: {diff --git a/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.ts b/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.tsindex c0083b84e3..3433efa358 100644--- a/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.ts+++ b/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.ts@@ -5,7 +5,6 @@ import {type FilterState,type ModelConfig,type EvalTemplateSourceCodeLanguage,- type EvalTemplateType,type EvaluatorPromptMessage,} from "@langfuse/shared";import { createStore, type StoreApi } from "zustand/vanilla";@@ -21,6 +20,8 @@ import type {import type { JudgeModel } from "@/src/features/evals/v2/judgeModel";import type { ScoreOutputFormState } from "@/src/features/evals/v2/scoreOutputTypes";import type { NormalizedEvaluatorDefinition } from "@/src/features/evals/v2/server/evaluators/evaluatorTypes";++type EditableEvaluatorType = NormalizedEvaluatorDefinition["type"];import { toScoreOutputFormState } from "@/src/features/evals/v2/fns/scoreOutput/toScoreOutputFormState";import { questionsToDrafts } from "@/src/features/evals/v2/fns/evaluators/decisionModelQuestions";import type { DecisionModelQuestionDraft } from "@/src/features/evals/v2/types/decisionModel";@@ -91,7 +92,7 @@ function buildInitialQuestions(}type EvaluatorSetupStoreActions = {- setType: (type: EvalTemplateType) => void;+ setType: (type: EditableEvaluatorType) => void;setQuestion: (question: DecisionModelQuestionDraft) => void;addQuestion: (question: DecisionModelQuestionDraft) => void;removeQuestion: (id: string) => void;@@ -130,7 +131,7 @@ type EvaluatorSetupStoreActions = {export type EvaluatorSetupStoreState = {initialDefinition: NormalizedEvaluatorDefinition | undefined;- type: EvalTemplateType;+ type: EditableEvaluatorType;promptMessages: EvaluatorPromptMessage[];/** Stable client-only ids used by drag-and-drop; never persisted. */promptMessageIds: string[];@@ -185,7 +186,7 @@ export function createEvaluatorSetupStore({definition: NormalizedEvaluatorDefinition;} | null;initialSampleFilter?: FilterState;- initialType?: EvalTemplateType;+ initialType?: EditableEvaluatorType;defaultModel?: JudgeModel | null;mode: "create" | "edit";}): EvaluatorSetupStore {diff --git a/web/src/features/mcp/server/evals/tools/evaluatorInput.ts b/web/src/features/mcp/server/evals/tools/evaluatorInput.tsindex e0b0ae452b..40a108858e 100644--- a/web/src/features/mcp/server/evals/tools/evaluatorInput.ts+++ b/web/src/features/mcp/server/evals/tools/evaluatorInput.ts@@ -78,7 +78,7 @@ const McpEvalOutputDefinitionSchema = z.object({export const McpEvaluatorInputBase = z.object({name: CreateEvaluatorSchema.shape.name,description: CreateEvaluatorSchema.shape.description.unwrap().optional(),- type: z.enum(EvalTemplateType),+ type: z.enum([EvalTemplateType.LLM_AS_JUDGE, EvalTemplateType.CODE]),prompt: z.string().min(1).optional(),modelConfig: McpEvaluatorModelConfigSchema.optional().describe("Optional custom model configuration. Omit to use the project default model.",diff --git a/worker/src/features/evaluation/observationEval/createSchedulerDeps.ts b/worker/src/features/evaluation/observationEval/createSchedulerDeps.tsindex c9f3904629..18f8409cf3 100644--- a/worker/src/features/evaluation/observationEval/createSchedulerDeps.ts+++ b/worker/src/features/evaluation/observationEval/createSchedulerDeps.ts@@ -106,6 +106,13 @@ export function createObservationEvalSchedulerDeps(): ObservationEvalSchedulerDereturn;}+ if (+ params.evalTemplateType !== EvalTemplateType.LLM_AS_JUDGE &&+ params.evalTemplateType !== EvalTemplateType.DECISION_MODEL+ ) {+ throw new Error("Evaluator type cannot use observation eval queues");+ }+const queue = LLMAsJudgeExecutionQueue.getInstance({ shardingKey });if (!queue) {throw new Error("LLMAsJudgeExecutionQueue is not initialized");diff --git a/worker/src/features/evaluation/observationEval/fetchObservationEvalRules.ts b/worker/src/features/evaluation/observationEval/fetchObservationEvalRules.tsindex a2a83064a8..f91dd53ae7 100644--- a/worker/src/features/evaluation/observationEval/fetchObservationEvalRules.ts+++ b/worker/src/features/evaluation/observationEval/fetchObservationEvalRules.ts@@ -1,4 +1,5 @@import {+ EvalTemplateType,EvalTargetObject,JobConfigState,coerceLegacyEmptyMetadataFilters,@@ -13,11 +14,17 @@ import {} from "@langfuse/shared/src/server";import { type ObservationEvalRule } from "./types";+const runnableEvaluatorTypes = [+ EvalTemplateType.LLM_AS_JUDGE,+ EvalTemplateType.CODE,+ EvalTemplateType.DECISION_MODEL,+];+/*** Fetches the runnable observation evaluation rules for a project.** This runs per ingested observation, so it stays as narrow as possible:- * inactive rules and blocked evaluators are excluded in SQL rather than+ * inactive rules and non-runnable evaluators are excluded in SQL rather than* filtered in the scheduler, and evaluator versions are not joined at all —* dispatch only needs the evaluator's identity and type, and the executor* resolves the definition when it picks the job up.@@ -49,9 +56,14 @@ export async function fetchObservationEvalRules(in: [EvalTargetObject.EVENT, EvalTargetObject.EXPERIMENT],},status: JobConfigState.ACTIVE,- // A rule whose every evaluator is blocked schedules nothing, so it must- // not keep the project out of the "no rules" cache below.- assignments: { some: { projectId, evaluator: { blockedAt: null } } },+ // A rule without runnable assignments schedules nothing, so it must not+ // keep the project out of the "no rules" cache below.+ assignments: {+ some: {+ projectId,+ evaluator: { blockedAt: null, type: { in: runnableEvaluatorTypes } },+ },+ },},select: {id: true,@@ -61,7 +73,10 @@ export async function fetchObservationEvalRules(status: true,targetObject: true,assignments: {- where: { projectId, evaluator: { blockedAt: null } },+ where: {+ projectId,+ evaluator: { blockedAt: null, type: { in: runnableEvaluatorTypes } },+ },select: {id: true,evaluatorId: true,diff --git a/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts b/worker/src/features/evaluation/observationEval/observationEvalProcessor.tsindex c0a87bd3ba..a707256450 100644--- a/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts+++ b/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts@@ -493,6 +493,8 @@ function normalizeEvalTemplate(sourceCodeLanguage: null,questions: template.questions,};+ case EvalTemplateType.FACET:+ throw new UnrecoverableError("Facets cannot run as evaluators");}}diff --git a/worker/src/features/evaluation/observationEval/scheduleObservationEvals.ts b/worker/src/features/evaluation/observationEval/scheduleObservationEvals.tsindex d9ed113ad6..032381858d 100644--- a/worker/src/features/evaluation/observationEval/scheduleObservationEvals.ts+++ b/worker/src/features/evaluation/observationEval/scheduleObservationEvals.ts@@ -15,6 +15,7 @@ import {logger,} from "@langfuse/shared/src/server";import {+ EvalTemplateType,JobExecutionStatus,type FilterState,type EvalExecutionMode,@@ -308,20 +309,24 @@ function getExecutableAssignments(rule: ObservationEvalRule,): ScheduledObservationEvalAssignment[] {if (!("assignments" in rule)) {- return rule.evalTemplateId- ? [- {- id: rule.id,- evaluatorId: null,- evaluationRuleId: null,- evalTemplateId: rule.evalTemplateId,- evaluatorType: rule.evalTemplate.type,- },- ]- : [];+ if (+ !rule.evalTemplateId ||+ rule.evalTemplate.type === EvalTemplateType.FACET+ )+ return [];+ return [+ {+ id: rule.id,+ evaluatorId: null,+ evaluationRuleId: null,+ evalTemplateId: rule.evalTemplateId,+ evaluatorType: rule.evalTemplate.type,+ },+ ];}return rule.assignments.flatMap((assignment) => {+ if (assignment.evaluator.type === EvalTemplateType.FACET) return [];// Blocked evaluators are already excluded by the query; this guards the// tenant boundary for callers that build assignments by hand.if (assignment.evaluator.projectId !== rule.projectId) {