🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(evals): surface decision-model evaluators in the template gallery (#17778)
Hassieb Pakzad17c3e9d97ca9
diff --git a/web/src/features/evals/server/legacyCompatibilityService.ts b/web/src/features/evals/server/legacyCompatibilityService.tsindex ce89a99a5f..dbbc97336a 100644--- a/web/src/features/evals/server/legacyCompatibilityService.ts+++ b/web/src/features/evals/server/legacyCompatibilityService.ts@@ -91,7 +91,25 @@ function managedTemplateId(key: string) {return `${MANAGED_TEMPLATE_ID_PREFIX}${key}`;}-function toLegacyManagedTemplate(template: ManagedTemplate) {+type LegacyManagedTemplate = ManagedTemplate & {+ evaluator: Exclude<+ ManagedTemplate["evaluator"],+ { type: typeof EvalTemplateType.DECISION_MODEL }+ >;+};++function isLegacyManagedTemplate(+ template: ManagedTemplate,+): template is LegacyManagedTemplate {+ return template.evaluator.type !== EvalTemplateType.DECISION_MODEL;+}++function legacyManagedTemplates(): LegacyManagedTemplate[] {+ const templates: ManagedTemplate[] = MANAGED_TEMPLATES_CATALOG.templates;+ return templates.filter(isLegacyManagedTemplate);+}++function toLegacyManagedTemplate(template: LegacyManagedTemplate) {const now = new Date(0);if (template.evaluator.type === EvalTemplateType.CODE) {return {@@ -203,7 +221,7 @@ type LlmEvaluatorVariableMapping = Extract<>["variableMapping"];function definitionFromManagedTemplate(- template: ManagedTemplate,+ template: LegacyManagedTemplate,variableMapping: LlmEvaluatorVariableMapping,): EvaluatorDefinition {if (template.evaluator.type === EvalTemplateType.CODE) {@@ -408,7 +426,7 @@ type ManagedCatalogEntry = {};function runnableManagedCatalog(): ManagedCatalogEntry[] {- return MANAGED_TEMPLATES_CATALOG.templates+ return legacyManagedTemplates().map((template) => ({template: toLegacyManagedTemplate(template),definition: definitionFromManagedTemplate(template, null),@@ -897,7 +915,7 @@ export class LegacyEvalCompatibilityService {}listManagedTemplates() {- return MANAGED_TEMPLATES_CATALOG.templates+ return legacyManagedTemplates().map(toLegacyManagedTemplate).filter(isRunnableTemplate);}@@ -921,7 +939,7 @@ export class LegacyEvalCompatibilityService {async getTemplate(projectId: string, templateId: string) {if (templateId.startsWith(MANAGED_TEMPLATE_ID_PREFIX)) {const key = templateId.slice(MANAGED_TEMPLATE_ID_PREFIX.length);- const template = MANAGED_TEMPLATES_CATALOG.templates.find(+ const template = legacyManagedTemplates().find((candidate) => candidate.key === key,);if (!template) return null;@@ -948,7 +966,7 @@ export class LegacyEvalCompatibilityService {}) {if (params.templateId.startsWith(MANAGED_TEMPLATE_ID_PREFIX)) {const key = params.templateId.slice(MANAGED_TEMPLATE_ID_PREFIX.length);- const template = MANAGED_TEMPLATES_CATALOG.templates.find(+ const template = legacyManagedTemplates().find((candidate) => candidate.key === key,);return template@@ -1234,9 +1252,7 @@ export class LegacyEvalCompatibilityService {? params.intent.cloneSourceId.slice(MANAGED_TEMPLATE_ID_PREFIX.length): null;const source = key- ? MANAGED_TEMPLATES_CATALOG.templates.find(- (candidate) => candidate.key === key,- )+ ? legacyManagedTemplates().find((candidate) => candidate.key === key): undefined;if (!source) {throw new LangfuseNotFoundError(diff --git a/web/src/features/evals/v2/components/EvaluatorGalleryDialog/EvaluatorGalleryDialog.tsx b/web/src/features/evals/v2/components/EvaluatorGalleryDialog/EvaluatorGalleryDialog.tsxindex a38ccebe1e..ce4a2d5db1 100644--- a/web/src/features/evals/v2/components/EvaluatorGalleryDialog/EvaluatorGalleryDialog.tsx+++ b/web/src/features/evals/v2/components/EvaluatorGalleryDialog/EvaluatorGalleryDialog.tsx@@ -12,6 +12,8 @@ import { EvaluatorGalleryView } from "@/src/features/evals/v2/components/Evaluatimport type { GalleryTemplate } from "@/src/features/evals/v2/types/templateGallery";import { prepareEvaluatorGallery } from "@/src/features/evals/v2/fns/templateGallery/prepareEvaluatorGallery";import { EVALUATOR_GALLERY_ALL_SECTION_KEY } from "@/src/features/evals/v2/constants/evaluatorGallery";+import useIsFeatureEnabled from "@/src/features/feature-flags/hooks/useIsFeatureEnabled";+import useLocalStorage from "@/src/components/useLocalStorage";import { usePostHogClientCapture } from "@/src/features/posthog-analytics";import { getEvaluatorCreationAnalyticsProperties } from "@/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties";import { api } from "@/src/utils/api";@@ -39,6 +41,13 @@ export function EvaluatorGalleryDialog({);const searchInputRef = useRef<HTMLInputElement | null>(null);const scrollContainerRef = useRef<HTMLDivElement | null>(null);+ const decisionModelsEnabled = useIsFeatureEnabled("decisionModelEvaluators", {+ projectId,+ });+ const [bannerDismissed, setBannerDismissed] = useLocalStorage(+ "evaluatorGallery:decisionModelBannerDismissed:v1",+ false,+ );const projectEvaluators = api.evalsV2.listGallery.useInfiniteQuery({projectId,@@ -75,6 +84,7 @@ export function EvaluatorGalleryDialog({customTemplateCount:projectEvaluators.data?.pages[0]?.totalItems ?? customTemplates.length,search,+ includeDecisionModels: decisionModelsEnabled,});const selectSection = (key: string) => {setActiveSection(key);@@ -154,6 +164,14 @@ export function EvaluatorGalleryDialog({? projectEvaluators.error.message: undefined}+ decisionModel={+ decisionModelsEnabled+ ? {+ bannerDismissed,+ onDismissBanner: () => setBannerDismissed(true),+ }+ : undefined+ }/></DialogContent></Dialog>diff --git a/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.stories.tsx b/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.stories.tsxindex 2ae670b006..1fe0a48152 100644--- a/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.stories.tsx+++ b/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.stories.tsx@@ -202,6 +202,62 @@ export const Default = meta.story({render: StatefulEvaluatorGalleryView,});+const topicDecisionModel = {+ source: "managed",+ key: "topic-decision-model",+ name: "Assign Input Topic",+ categories: ["classifier", "recommended"],+ icon: "tags",+ description:+ "Assigns the input to one of your topics with a calibrated probability for each option, in one fast decision-model call.",+ maintainer: "langfuse",+ evaluator: {+ type: EvalTemplateTypeEnum.DECISION_MODEL,+ questions: [+ {+ id: "topic",+ type: "choice",+ scoreName: "topic",+ instructions: "Which topic does `input` belong to?",+ options: [+ { value: "support", description: "Product help or troubleshooting." },+ { value: "billing", description: "Invoices, pricing, payments." },+ { value: "sales", description: "Purchase, trial, demo, enterprise." },+ { value: "other", description: "None of the above." },+ ],+ },+ ],+ state: [{ key: "input", defaultMapping: { field: "input" } }],+ },+} satisfies GalleryTemplate;++const sectionsWithDecisionModelCard = sections.map((section) =>+ section.key === "recommended"+ ? {+ ...section,+ templates: [topicDecisionModel, ...section.templates.slice(0, 2)],+ }+ : section,+);++export const DecisionModels = meta.story({+ args: {+ ...defaultArgs,+ sections: sectionsWithDecisionModelCard,+ decisionModel: { bannerDismissed: false, onDismissBanner: fn() },+ },+ render: StatefulEvaluatorGalleryView,+});++export const DecisionModelsBannerDismissed = meta.story({+ args: {+ ...defaultArgs,+ sections: sectionsWithDecisionModelCard,+ decisionModel: { bannerDismissed: true, onDismissBanner: fn() },+ },+ render: StatefulEvaluatorGalleryView,+});+export const YourTemplates = meta.story({args: {...defaultArgs,diff --git a/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.tsx b/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.tsxindex 328233ba5a..0b954a0eaf 100644--- a/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.tsx+++ b/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.tsx@@ -1,11 +1,12 @@/* eslint-disable no-nested-ternary */import { useCallback, type RefObject } from "react";-import { Code2, Search, Sparkles } from "lucide-react";+import { Code2, Scale, Search, Sparkles } from "lucide-react";import { EvalTemplateTypeEnum, type EvalTemplateType } from "@langfuse/shared";import { Button } from "@/src/components/ui/button";import { Input } from "@/src/components/ui/input";import { Skeleton } from "@/src/components/ui/skeleton";+import { EvaluatorGalleryDecisionModelBanner } from "./components/EvaluatorGalleryDecisionModelBanner/EvaluatorGalleryDecisionModelBanner";import { EvaluatorGallerySection } from "./components/EvaluatorGallerySection/EvaluatorGallerySection";import { EvaluatorGallerySidebar } from "./components/EvaluatorGallerySidebar/EvaluatorGallerySidebar";import type {@@ -53,6 +54,7 @@ export function EvaluatorGalleryView({isLoadingMoreProjectTemplates = false,onLoadMoreProjectTemplates,errorMessage,+ decisionModel,}: {search: string;onSearchChange: (search: string) => void;@@ -71,6 +73,10 @@ export function EvaluatorGalleryView({isLoadingMoreProjectTemplates?: boolean;onLoadMoreProjectTemplates?: () => void;errorMessage?: string;+ decisionModel?: {+ bannerDismissed: boolean;+ onDismissBanner: () => void;+ };}) {const sidebarItems = gallerySidebarItems(navigationItems, sections);const resolvedSection =@@ -169,10 +175,32 @@ export function EvaluatorGalleryView({<Code2 className="h-3.5 w-3.5" aria-hidden="true" />New code evaluator</Button>+ {decisionModel ? (+ <Button+ type="button"+ variant="secondary"+ className="flex-1 shrink-0 gap-1.5 @2xl:flex-none"+ title="Ask TypeSafe Jev typed questions and get calibrated answers in one call. Experimental."+ onClick={() =>+ onCreateFromScratch(EvalTemplateTypeEnum.DECISION_MODEL)+ }+ >+ <Scale className="h-3.5 w-3.5" aria-hidden="true" />+ New decision model evaluator+ </Button>+ ) : null}</div></div><div className="flex flex-col gap-10 px-4 py-4">+ {decisionModel && !decisionModel.bannerDismissed ? (+ <EvaluatorGalleryDecisionModelBanner+ onTry={() =>+ onCreateFromScratch(EvalTemplateTypeEnum.DECISION_MODEL)+ }+ onDismiss={decisionModel.onDismissBanner}+ />+ ) : null}{isLoading ? <GallerySkeleton /> : null}{errorMessage ? (<div className="text-destructive py-8 text-center text-sm">diff --git a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryDecisionModelBanner/EvaluatorGalleryDecisionModelBanner.tsx b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryDecisionModelBanner/EvaluatorGalleryDecisionModelBanner.tsxnew file mode 100644index 0000000000..9793125e8b--- /dev/null+++ b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryDecisionModelBanner/EvaluatorGalleryDecisionModelBanner.tsx@@ -0,0 +1,40 @@+import { Scale, X } from "lucide-react";++import { Button } from "@/src/components/ui/button";++export function EvaluatorGalleryDecisionModelBanner({+ onTry,+ onDismiss,+}: {+ onTry: () => void;+ onDismiss: () => void;+}) {+ return (+ <div className="border-primary-accent/30 bg-primary-accent/5 flex flex-wrap items-center gap-3 rounded-md border px-4 py-3 text-sm">+ <Scale+ className="text-primary-accent h-4 w-4 shrink-0"+ aria-hidden="true"+ />+ <div className="min-w-0 flex-1">+ <span className="font-bold">New: decision-model evaluators.</span>{" "}+ <span className="text-muted-foreground">+ Ask TypeSafe Jev typed questions about each observation and get+ calibrated answers in one call, at a fraction of an LLM judge's+ cost. Experimental.+ </span>+ </div>+ <Button type="button" size="sm" onClick={onTry}>+ Try a decision model+ </Button>+ <Button+ type="button"+ variant="ghost"+ size="icon-xs"+ aria-label="Dismiss"+ onClick={onDismiss}+ >+ <X className="h-3.5 w-3.5" />+ </Button>+ </div>+ );+}diff --git a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryMethodBadge/EvaluatorGalleryMethodBadge.tsx b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryMethodBadge/EvaluatorGalleryMethodBadge.tsxindex 60074c07fc..129d111d87 100644--- a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryMethodBadge/EvaluatorGalleryMethodBadge.tsx+++ b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryMethodBadge/EvaluatorGalleryMethodBadge.tsx@@ -3,12 +3,15 @@ import { EvalTemplateTypeEnum, type EvalTemplateType } from "@langfuse/shared";const METHOD_BADGES = {[EvalTemplateTypeEnum.CODE]: { Icon: Code2, label: "Code" },- [EvalTemplateTypeEnum.LLM_AS_JUDGE]: { Icon: Sparkles, label: "LLM judge" },+ [EvalTemplateTypeEnum.LLM_AS_JUDGE]: {+ Icon: Sparkles,+ label: "LLM judge",+ },[EvalTemplateTypeEnum.DECISION_MODEL]: {Icon: Scale,label: "Decision model",},-} as const satisfies Record<EvalTemplateType, unknown>;+} as const satisfies Record<EvalTemplateType, { Icon: unknown; label: string }>;export function EvaluatorGalleryMethodBadge({type,@@ -18,7 +21,7 @@ export function EvaluatorGalleryMethodBadge({const { Icon, label } = METHOD_BADGES[type];return (- <span className="bg-muted text-muted-foreground inline-flex h-5 shrink-0 items-center gap-1 rounded-full px-2 text-xs leading-none">+ <span className="bg-muted text-muted-foreground inline-flex h-5 shrink-0 items-center gap-1 rounded-full px-2 text-xs leading-none whitespace-nowrap"><Icon className="size-3" />{label}</span>diff --git a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/components/EvaluatorTemplateRow/EvaluatorTemplateRow.tsx b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/components/EvaluatorTemplateRow/EvaluatorTemplateRow.tsxindex 37b059d0c7..feede29547 100644--- a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/components/EvaluatorTemplateRow/EvaluatorTemplateRow.tsx+++ b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/components/EvaluatorTemplateRow/EvaluatorTemplateRow.tsx@@ -18,7 +18,7 @@ export function EvaluatorTemplateRow({onClick={() => onSelect(template)}className="hover:bg-muted/60 @container block w-full cursor-pointer rounded-md px-1 py-2.5 text-left transition-colors">- <span className="grid w-full grid-cols-[minmax(0,1fr)_auto] items-center gap-x-3 gap-y-1 @3xl:grid-cols-[minmax(10rem,18rem)_minmax(0,1fr)_max-content_6rem]">+ <span className="grid w-full grid-cols-[minmax(0,1fr)_auto] items-center gap-x-3 gap-y-1 @3xl:grid-cols-[minmax(10rem,18rem)_minmax(0,1fr)_max-content_max-content]"><spanclassName="col-start-1 row-start-1 min-w-0 truncate text-sm leading-5 font-bold"title={template.name}diff --git a/web/src/features/evals/v2/constants/managedTemplatesCatalog.clienttest.ts b/web/src/features/evals/v2/constants/managedTemplatesCatalog.clienttest.tsindex c8e7f9ceec..1827c18b50 100644--- a/web/src/features/evals/v2/constants/managedTemplatesCatalog.clienttest.ts+++ b/web/src/features/evals/v2/constants/managedTemplatesCatalog.clienttest.ts@@ -16,7 +16,7 @@ describe("managed evaluator templates catalog", () => {({ key }) => key,);- expect(MANAGED_TEMPLATES_CATALOG.templates).toHaveLength(20);+ expect(MANAGED_TEMPLATES_CATALOG.templates).toHaveLength(23);expect(new Set(templateKeys).size).toBe(templateKeys.length);for (const template of MANAGED_TEMPLATES_CATALOG.templates) {diff --git a/web/src/features/evals/v2/constants/managedTemplatesCatalog.ts b/web/src/features/evals/v2/constants/managedTemplatesCatalog.tsindex a046e827f0..42528d3371 100644--- a/web/src/features/evals/v2/constants/managedTemplatesCatalog.ts+++ b/web/src/features/evals/v2/constants/managedTemplatesCatalog.ts@@ -1434,5 +1434,118 @@ Task text: {{task_text}}`,},},},+ {+ key: "topic-decision-model",+ name: "Assign Input Topic",+ categories: ["classifier", "recommended"],+ icon: "tags",+ description:+ "Assigns the input to one of your topics with a calibrated probability for each option, in one fast decision-model call.",+ maintainer: "langfuse",+ evaluator: {+ type: "DECISION_MODEL",+ questions: [+ {+ id: "topic",+ type: "choice",+ scoreName: "topic",+ instructions:+ "Which topic best matches the user's primary goal in `input`? Replace the options below with your own taxonomy before use.",+ options: [+ {+ value: "support",+ description: "Asks for product help or troubleshooting.",+ },+ {+ value: "billing",+ description:+ "Asks about invoices, pricing, payments, or subscriptions.",+ },+ {+ value: "technical",+ description: "Asks technical implementation questions.",+ },+ {+ value: "sales",+ description:+ "Asks about purchase, trial, demo, or enterprise fit.",+ },+ {+ value: "feedback",+ description: "Shares feature feedback or product suggestions.",+ },+ {+ value: "other",+ description: "Does not reasonably fit any topic above.",+ },+ ],+ },+ ],+ state: [{ key: "input", defaultMapping: { field: "input" } }],+ },+ },+ {+ key: "out-of-scope-decision-model",+ name: "Flag Out-of-Scope Request",+ categories: ["conversation"],+ icon: "shield",+ description:+ "Returns the probability that the user's request falls outside what the assistant is meant to handle.",+ maintainer: "langfuse",+ evaluator: {+ type: "DECISION_MODEL",+ questions: [+ {+ id: "out_of_scope",+ type: "noul",+ scoreName: "out_of_scope",+ instructions:+ "Does `input` ask for something outside the assistant's role or supported scope, judging from how the assistant presents itself in `output`?",+ criteria: {+ true: "The request concerns an unrelated task, another product, or a personal favor the assistant is not meant to handle.",+ false:+ "The request is about the assistant's product or role, even if phrased vaguely or partially answerable.",+ },+ },+ ],+ state: [+ { key: "input", defaultMapping: { field: "input" } },+ { key: "output", defaultMapping: { field: "output" } },+ ],+ },+ },+ {+ key: "frustration-decision-model",+ name: "Rate Customer Frustration",+ categories: ["conversation"],+ icon: "frown",+ description:+ "Rates how frustrated the user sounds along levels you describe, from calm to ready to leave.",+ maintainer: "langfuse",+ evaluator: {+ type: "DECISION_MODEL",+ questions: [+ {+ id: "frustration",+ type: "score",+ scoreName: "customer_frustration",+ instructions: "How frustrated is the user in `input`?",+ levels: [+ { description: "Calm, neutral, or friendly." },+ { description: "Mildly annoyed but civil." },+ {+ description:+ "Clearly frustrated: repeating themselves, complaining, or using sharp language.",+ },+ {+ description:+ "Angry: insulting, threatening to leave, or demanding escalation.",+ },+ ],+ },+ ],+ state: [{ key: "input", defaultMapping: { field: "input" } }],+ },+ },],} satisfies ManagedTemplatesCatalog;diff --git a/web/src/features/evals/v2/fns/templateGallery/managedEvaluatorTemplateService.ts b/web/src/features/evals/v2/fns/templateGallery/managedEvaluatorTemplateService.tsindex 79b263cf7b..0052f0957f 100644--- a/web/src/features/evals/v2/fns/templateGallery/managedEvaluatorTemplateService.ts+++ b/web/src/features/evals/v2/fns/templateGallery/managedEvaluatorTemplateService.ts@@ -5,10 +5,12 @@ export const managedEvaluatorTemplateService = {search,category,type,+ includeDecisionModels = false,}: {search?: string;category?: string;type?: "LLM_AS_JUDGE" | "CODE";+ includeDecisionModels?: boolean;} = {}) {const query = search?.trim().toLowerCase();@@ -17,6 +19,8 @@ export const managedEvaluatorTemplateService = {categories: MANAGED_TEMPLATES_CATALOG.categories,templates: MANAGED_TEMPLATES_CATALOG.templates.filter((template) =>+ (includeDecisionModels ||+ template.evaluator.type !== "DECISION_MODEL") &&(!query ||template.name.toLowerCase().includes(query) ||template.description.toLowerCase().includes(query)) &&diff --git a/web/src/features/evals/v2/fns/templateGallery/managedTemplateToEvaluatorSetupDraft.ts b/web/src/features/evals/v2/fns/templateGallery/managedTemplateToEvaluatorSetupDraft.tsindex 7872d77701..6ee2dd62cd 100644--- a/web/src/features/evals/v2/fns/templateGallery/managedTemplateToEvaluatorSetupDraft.ts+++ b/web/src/features/evals/v2/fns/templateGallery/managedTemplateToEvaluatorSetupDraft.ts@@ -3,36 +3,55 @@ import type {ManagedTemplate,} from "@/src/features/evals/v2/types/templateGallery";+function managedTemplateDefinition(+ evaluator: ManagedTemplate["evaluator"],+): EvaluatorSetupDraft["definition"] {+ switch (evaluator.type) {+ case "LLM_AS_JUDGE":+ return {+ type: evaluator.type,+ promptMessages: evaluator.promptMessages,+ provider: null,+ model: null,+ modelParams: null,+ vars: evaluator.variables.map(({ name }) => name),+ variableMapping: evaluator.variables.map(+ ({ name, defaultMapping }) => ({+ templateVariable: name,+ selectedColumnId: defaultMapping.field,+ jsonSelector: null,+ }),+ ),+ outputDefinition: evaluator.outputDefinition,+ };+ case "CODE":+ return {+ type: evaluator.type,+ sourceCode: evaluator.source,+ sourceCodeLanguage: evaluator.language,+ };+ case "DECISION_MODEL":+ return {+ type: evaluator.type,+ questions: evaluator.questions,+ provider: "",+ model: "",+ vars: evaluator.state.map(({ key }) => key),+ variableMapping: evaluator.state.map(({ key, defaultMapping }) => ({+ templateVariable: key,+ selectedColumnId: defaultMapping.field,+ jsonSelector: null,+ })),+ };+ }+}+export function managedTemplateToEvaluatorSetupDraft(template: ManagedTemplate,): EvaluatorSetupDraft {- const definition =- template.evaluator.type === "LLM_AS_JUDGE"- ? {- type: template.evaluator.type,- promptMessages: template.evaluator.promptMessages,- provider: null,- model: null,- modelParams: null,- vars: template.evaluator.variables.map(({ name }) => name),- variableMapping: template.evaluator.variables.map(- ({ name, defaultMapping }) => ({- templateVariable: name,- selectedColumnId: defaultMapping.field,- jsonSelector: null,- }),- ),- outputDefinition: template.evaluator.outputDefinition,- }- : {- type: template.evaluator.type,- sourceCode: template.evaluator.source,- sourceCodeLanguage: template.evaluator.language,- };-return {name: template.name,description: template.description,- definition,+ definition: managedTemplateDefinition(template.evaluator),};}diff --git a/web/src/features/evals/v2/fns/templateGallery/prepareEvaluatorEmptyState.clienttest.ts b/web/src/features/evals/v2/fns/templateGallery/prepareEvaluatorEmptyState.clienttest.tsindex fd8fc7fcf4..84c4e07996 100644--- a/web/src/features/evals/v2/fns/templateGallery/prepareEvaluatorEmptyState.clienttest.ts+++ b/web/src/features/evals/v2/fns/templateGallery/prepareEvaluatorEmptyState.clienttest.ts@@ -11,7 +11,9 @@ describe("prepareEvaluatorEmptyState", () => {expect(emptyState.docsHref).toBe(EVALUATOR_EMPTY_STATE_DOCS_HREF);expect(emptyState.templateCount).toBe(- MANAGED_TEMPLATES_CATALOG.templates.length,+ MANAGED_TEMPLATES_CATALOG.templates.filter(+ (template) => template.evaluator.type !== "DECISION_MODEL",+ ).length,);expect(emptyState.startingPoints.map(({ template, action }) => ({diff --git a/web/src/features/evals/v2/fns/templateGallery/prepareEvaluatorEmptyState.ts b/web/src/features/evals/v2/fns/templateGallery/prepareEvaluatorEmptyState.tsindex 498e91581a..94d173d33e 100644--- a/web/src/features/evals/v2/fns/templateGallery/prepareEvaluatorEmptyState.ts+++ b/web/src/features/evals/v2/fns/templateGallery/prepareEvaluatorEmptyState.ts@@ -1,4 +1,3 @@-import { MANAGED_TEMPLATES_CATALOG } from "@/src/features/evals/v2/constants/managedTemplatesCatalog";import {EVALUATOR_EMPTY_STATE_DOCS_HREF,EVALUATOR_EMPTY_STATE_STARTING_POINTS,@@ -32,7 +31,7 @@ export function prepareEvaluatorEmptyState(): EvaluatorEmptyStateModel {const template = managedEvaluatorTemplateService.get(point.templateKey);return template ? [toStartingPoint(point, template)] : [];}),- templateCount: MANAGED_TEMPLATES_CATALOG.templates.length,+ templateCount: managedEvaluatorTemplateService.list().templates.length,docsHref: EVALUATOR_EMPTY_STATE_DOCS_HREF,};}diff --git a/web/src/features/evals/v2/fns/templateGallery/prepareEvaluatorGallery.ts b/web/src/features/evals/v2/fns/templateGallery/prepareEvaluatorGallery.tsindex e125703f60..c50c5fa470 100644--- a/web/src/features/evals/v2/fns/templateGallery/prepareEvaluatorGallery.ts+++ b/web/src/features/evals/v2/fns/templateGallery/prepareEvaluatorGallery.ts@@ -8,11 +8,16 @@ import { managedEvaluatorTemplateService } from "@/src/features/evals/v2/fns/temimport { EVALUATOR_GALLERY_PROJECT_SECTION_KEY } from "@/src/features/evals/v2/constants/evaluatorGallery";const RECOMMENDED_TEMPLATE_ORDER = [+ "topic-decision-model","topic-classifier","out-of-scope-request","quality-criterion",] as const;+const RECOMMENDED_TEMPLATE_REPLACEMENTS: Record<string, string> = {+ "topic-decision-model": "topic-classifier",+};+const RECOMMENDED_TEMPLATE_RANK = new Map<string, number>(RECOMMENDED_TEMPLATE_ORDER.map((key, index) => [key, index]),);@@ -21,10 +26,12 @@ export function prepareEvaluatorGallery({customTemplates,customTemplateCount,search,+ includeDecisionModels = false,}: {customTemplates: CustomEvaluatorTemplate[];customTemplateCount: number;search: string;+ includeDecisionModels?: boolean;}): {navigationItems: GalleryNavigationItem[];sections: GallerySection[];@@ -37,21 +44,31 @@ export function prepareEvaluatorGallery({const filteredCustom = customTemplates.filter((template) =>matches(template.name, template.description),);- const managedCatalog = managedEvaluatorTemplateService.list({ search });+ const managedCatalog = managedEvaluatorTemplateService.list({+ search,+ includeDecisionModels,+ });const managedByCategory = new Map(managedCatalog.categories.map((category) => {const templatesInCategory = managedCatalog.templates.filter((template) =>template.categories.includes(category.key),);+ const replacedKeys = new Set(+ templatesInCategory.flatMap(+ (template) => RECOMMENDED_TEMPLATE_REPLACEMENTS[template.key] ?? [],+ ),+ );const orderedTemplates =category.key === "recommended"- ? templatesInCategory.toSorted(- (left, right) =>- (RECOMMENDED_TEMPLATE_RANK.get(left.key) ??- Number.MAX_SAFE_INTEGER) -- (RECOMMENDED_TEMPLATE_RANK.get(right.key) ??- Number.MAX_SAFE_INTEGER),- )+ ? templatesInCategory+ .filter((template) => !replacedKeys.has(template.key))+ .toSorted(+ (left, right) =>+ (RECOMMENDED_TEMPLATE_RANK.get(left.key) ??+ Number.MAX_SAFE_INTEGER) -+ (RECOMMENDED_TEMPLATE_RANK.get(right.key) ??+ Number.MAX_SAFE_INTEGER),+ ): templatesInCategory;return [diff --git a/web/src/features/evals/v2/types/templateGallery.ts b/web/src/features/evals/v2/types/templateGallery.tsindex 9321b5d669..2c9839e7f9 100644--- a/web/src/features/evals/v2/types/templateGallery.ts+++ b/web/src/features/evals/v2/types/templateGallery.ts@@ -1,5 +1,6 @@import type { LucideIcon } from "lucide-react";import type {+ DecisionModelQuestions,EvalTemplateSourceCodeLanguage,EvalTemplateType,EvaluatorPromptMessage,@@ -35,6 +36,11 @@ export type ManagedTemplate = {type: Extract<EvalTemplateType, "CODE">;language: EvalTemplateSourceCodeLanguage;source: string;+ }+ | {+ type: Extract<EvalTemplateType, "DECISION_MODEL">;+ questions: DecisionModelQuestions;+ state: Array<{ key: string; defaultMapping: { field: string } }>;};};