Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(evals): surface decision-model evaluators in the template gallery (#17778)

Hassieb Pakzad17c3e9d97ca9
diff --git a/web/src/features/evals/server/legacyCompatibilityService.ts b/web/src/features/evals/server/legacyCompatibilityService.ts
index ce89a99a5f..dbbc97336a 100644
--- a/web/src/features/evals/server/legacyCompatibilityService.ts
+++ b/web/src/features/evals/server/legacyCompatibilityService.ts
@@ -91,7 +91,25 @@ function managedTemplateId(key: string) {
return `${MANAGED_TEMPLATE_ID_PREFIX}${key}`;
}
-function toLegacyManagedTemplate(template: ManagedTemplate) {
+type LegacyManagedTemplate = ManagedTemplate & {
+ evaluator: Exclude<
+ ManagedTemplate["evaluator"],
+ { type: typeof EvalTemplateType.DECISION_MODEL }
+ >;
+};
+
+function isLegacyManagedTemplate(
+ template: ManagedTemplate,
+): template is LegacyManagedTemplate {
+ return template.evaluator.type !== EvalTemplateType.DECISION_MODEL;
+}
+
+function legacyManagedTemplates(): LegacyManagedTemplate[] {
+ const templates: ManagedTemplate[] = MANAGED_TEMPLATES_CATALOG.templates;
+ return templates.filter(isLegacyManagedTemplate);
+}
+
+function toLegacyManagedTemplate(template: LegacyManagedTemplate) {
const now = new Date(0);
if (template.evaluator.type === EvalTemplateType.CODE) {
return {
@@ -203,7 +221,7 @@ type LlmEvaluatorVariableMapping = Extract<
>["variableMapping"];
function definitionFromManagedTemplate(
- template: ManagedTemplate,
+ template: LegacyManagedTemplate,
variableMapping: LlmEvaluatorVariableMapping,
): EvaluatorDefinition {
if (template.evaluator.type === EvalTemplateType.CODE) {
@@ -408,7 +426,7 @@ type ManagedCatalogEntry = {
};
function runnableManagedCatalog(): ManagedCatalogEntry[] {
- return MANAGED_TEMPLATES_CATALOG.templates
+ return legacyManagedTemplates()
.map((template) => ({
template: toLegacyManagedTemplate(template),
definition: definitionFromManagedTemplate(template, null),
@@ -897,7 +915,7 @@ export class LegacyEvalCompatibilityService {
}
listManagedTemplates() {
- return MANAGED_TEMPLATES_CATALOG.templates
+ return legacyManagedTemplates()
.map(toLegacyManagedTemplate)
.filter(isRunnableTemplate);
}
@@ -921,7 +939,7 @@ export class LegacyEvalCompatibilityService {
async getTemplate(projectId: string, templateId: string) {
if (templateId.startsWith(MANAGED_TEMPLATE_ID_PREFIX)) {
const key = templateId.slice(MANAGED_TEMPLATE_ID_PREFIX.length);
- const template = MANAGED_TEMPLATES_CATALOG.templates.find(
+ const template = legacyManagedTemplates().find(
(candidate) => candidate.key === key,
);
if (!template) return null;
@@ -948,7 +966,7 @@ export class LegacyEvalCompatibilityService {
}) {
if (params.templateId.startsWith(MANAGED_TEMPLATE_ID_PREFIX)) {
const key = params.templateId.slice(MANAGED_TEMPLATE_ID_PREFIX.length);
- const template = MANAGED_TEMPLATES_CATALOG.templates.find(
+ const template = legacyManagedTemplates().find(
(candidate) => candidate.key === key,
);
return template
@@ -1234,9 +1252,7 @@ export class LegacyEvalCompatibilityService {
? params.intent.cloneSourceId.slice(MANAGED_TEMPLATE_ID_PREFIX.length)
: null;
const source = key
- ? MANAGED_TEMPLATES_CATALOG.templates.find(
- (candidate) => candidate.key === key,
- )
+ ? legacyManagedTemplates().find((candidate) => candidate.key === key)
: undefined;
if (!source) {
throw new LangfuseNotFoundError(
diff --git a/web/src/features/evals/v2/components/EvaluatorGalleryDialog/EvaluatorGalleryDialog.tsx b/web/src/features/evals/v2/components/EvaluatorGalleryDialog/EvaluatorGalleryDialog.tsx
index a38ccebe1e..ce4a2d5db1 100644
--- a/web/src/features/evals/v2/components/EvaluatorGalleryDialog/EvaluatorGalleryDialog.tsx
+++ b/web/src/features/evals/v2/components/EvaluatorGalleryDialog/EvaluatorGalleryDialog.tsx
@@ -12,6 +12,8 @@ import { EvaluatorGalleryView } from "@/src/features/evals/v2/components/Evaluat
import type { GalleryTemplate } from "@/src/features/evals/v2/types/templateGallery";
import { prepareEvaluatorGallery } from "@/src/features/evals/v2/fns/templateGallery/prepareEvaluatorGallery";
import { EVALUATOR_GALLERY_ALL_SECTION_KEY } from "@/src/features/evals/v2/constants/evaluatorGallery";
+import useIsFeatureEnabled from "@/src/features/feature-flags/hooks/useIsFeatureEnabled";
+import useLocalStorage from "@/src/components/useLocalStorage";
import { usePostHogClientCapture } from "@/src/features/posthog-analytics";
import { getEvaluatorCreationAnalyticsProperties } from "@/src/features/evals/v2/fns/evaluators/getEvaluatorCreationAnalyticsProperties";
import { api } from "@/src/utils/api";
@@ -39,6 +41,13 @@ export function EvaluatorGalleryDialog({
);
const searchInputRef = useRef<HTMLInputElement | null>(null);
const scrollContainerRef = useRef<HTMLDivElement | null>(null);
+ const decisionModelsEnabled = useIsFeatureEnabled("decisionModelEvaluators", {
+ projectId,
+ });
+ const [bannerDismissed, setBannerDismissed] = useLocalStorage(
+ "evaluatorGallery:decisionModelBannerDismissed:v1",
+ false,
+ );
const projectEvaluators = api.evalsV2.listGallery.useInfiniteQuery(
{
projectId,
@@ -75,6 +84,7 @@ export function EvaluatorGalleryDialog({
customTemplateCount:
projectEvaluators.data?.pages[0]?.totalItems ?? customTemplates.length,
search,
+ includeDecisionModels: decisionModelsEnabled,
});
const selectSection = (key: string) => {
setActiveSection(key);
@@ -154,6 +164,14 @@ export function EvaluatorGalleryDialog({
? projectEvaluators.error.message
: undefined
}
+ decisionModel={
+ decisionModelsEnabled
+ ? {
+ bannerDismissed,
+ onDismissBanner: () => setBannerDismissed(true),
+ }
+ : undefined
+ }
/>
</DialogContent>
</Dialog>
diff --git a/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.stories.tsx b/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.stories.tsx
index 2ae670b006..1fe0a48152 100644
--- a/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.stories.tsx
+++ b/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.stories.tsx
@@ -202,6 +202,62 @@ export const Default = meta.story({
render: StatefulEvaluatorGalleryView,
});
+const topicDecisionModel = {
+ source: "managed",
+ key: "topic-decision-model",
+ name: "Assign Input Topic",
+ categories: ["classifier", "recommended"],
+ icon: "tags",
+ description:
+ "Assigns the input to one of your topics with a calibrated probability for each option, in one fast decision-model call.",
+ maintainer: "langfuse",
+ evaluator: {
+ type: EvalTemplateTypeEnum.DECISION_MODEL,
+ questions: [
+ {
+ id: "topic",
+ type: "choice",
+ scoreName: "topic",
+ instructions: "Which topic does `input` belong to?",
+ options: [
+ { value: "support", description: "Product help or troubleshooting." },
+ { value: "billing", description: "Invoices, pricing, payments." },
+ { value: "sales", description: "Purchase, trial, demo, enterprise." },
+ { value: "other", description: "None of the above." },
+ ],
+ },
+ ],
+ state: [{ key: "input", defaultMapping: { field: "input" } }],
+ },
+} satisfies GalleryTemplate;
+
+const sectionsWithDecisionModelCard = sections.map((section) =>
+ section.key === "recommended"
+ ? {
+ ...section,
+ templates: [topicDecisionModel, ...section.templates.slice(0, 2)],
+ }
+ : section,
+);
+
+export const DecisionModels = meta.story({
+ args: {
+ ...defaultArgs,
+ sections: sectionsWithDecisionModelCard,
+ decisionModel: { bannerDismissed: false, onDismissBanner: fn() },
+ },
+ render: StatefulEvaluatorGalleryView,
+});
+
+export const DecisionModelsBannerDismissed = meta.story({
+ args: {
+ ...defaultArgs,
+ sections: sectionsWithDecisionModelCard,
+ decisionModel: { bannerDismissed: true, onDismissBanner: fn() },
+ },
+ render: StatefulEvaluatorGalleryView,
+});
+
export const YourTemplates = meta.story({
args: {
...defaultArgs,
diff --git a/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.tsx b/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.tsx
index 328233ba5a..0b954a0eaf 100644
--- a/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.tsx
+++ b/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.tsx
@@ -1,11 +1,12 @@
/* eslint-disable no-nested-ternary */
import { useCallback, type RefObject } from "react";
-import { Code2, Search, Sparkles } from "lucide-react";
+import { Code2, Scale, Search, Sparkles } from "lucide-react";
import { EvalTemplateTypeEnum, type EvalTemplateType } from "@langfuse/shared";
import { Button } from "@/src/components/ui/button";
import { Input } from "@/src/components/ui/input";
import { Skeleton } from "@/src/components/ui/skeleton";
+import { EvaluatorGalleryDecisionModelBanner } from "./components/EvaluatorGalleryDecisionModelBanner/EvaluatorGalleryDecisionModelBanner";
import { EvaluatorGallerySection } from "./components/EvaluatorGallerySection/EvaluatorGallerySection";
import { EvaluatorGallerySidebar } from "./components/EvaluatorGallerySidebar/EvaluatorGallerySidebar";
import type {
@@ -53,6 +54,7 @@ export function EvaluatorGalleryView({
isLoadingMoreProjectTemplates = false,
onLoadMoreProjectTemplates,
errorMessage,
+ decisionModel,
}: {
search: string;
onSearchChange: (search: string) => void;
@@ -71,6 +73,10 @@ export function EvaluatorGalleryView({
isLoadingMoreProjectTemplates?: boolean;
onLoadMoreProjectTemplates?: () => void;
errorMessage?: string;
+ decisionModel?: {
+ bannerDismissed: boolean;
+ onDismissBanner: () => void;
+ };
}) {
const sidebarItems = gallerySidebarItems(navigationItems, sections);
const resolvedSection =
@@ -169,10 +175,32 @@ export function EvaluatorGalleryView({
<Code2 className="h-3.5 w-3.5" aria-hidden="true" />
New code evaluator
</Button>
+ {decisionModel ? (
+ <Button
+ type="button"
+ variant="secondary"
+ className="flex-1 shrink-0 gap-1.5 @2xl:flex-none"
+ title="Ask TypeSafe Jev typed questions and get calibrated answers in one call. Experimental."
+ onClick={() =>
+ onCreateFromScratch(EvalTemplateTypeEnum.DECISION_MODEL)
+ }
+ >
+ <Scale className="h-3.5 w-3.5" aria-hidden="true" />
+ New decision model evaluator
+ </Button>
+ ) : null}
</div>
</div>
<div className="flex flex-col gap-10 px-4 py-4">
+ {decisionModel && !decisionModel.bannerDismissed ? (
+ <EvaluatorGalleryDecisionModelBanner
+ onTry={() =>
+ onCreateFromScratch(EvalTemplateTypeEnum.DECISION_MODEL)
+ }
+ onDismiss={decisionModel.onDismissBanner}
+ />
+ ) : null}
{isLoading ? <GallerySkeleton /> : null}
{errorMessage ? (
<div className="text-destructive py-8 text-center text-sm">
diff --git a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryDecisionModelBanner/EvaluatorGalleryDecisionModelBanner.tsx b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryDecisionModelBanner/EvaluatorGalleryDecisionModelBanner.tsx
new file mode 100644
index 0000000000..9793125e8b
--- /dev/null
+++ b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryDecisionModelBanner/EvaluatorGalleryDecisionModelBanner.tsx
@@ -0,0 +1,40 @@
+import { Scale, X } from "lucide-react";
+
+import { Button } from "@/src/components/ui/button";
+
+export function EvaluatorGalleryDecisionModelBanner({
+ onTry,
+ onDismiss,
+}: {
+ onTry: () => void;
+ onDismiss: () => void;
+}) {
+ return (
+ <div className="border-primary-accent/30 bg-primary-accent/5 flex flex-wrap items-center gap-3 rounded-md border px-4 py-3 text-sm">
+ <Scale
+ className="text-primary-accent h-4 w-4 shrink-0"
+ aria-hidden="true"
+ />
+ <div className="min-w-0 flex-1">
+ <span className="font-bold">New: decision-model evaluators.</span>{" "}
+ <span className="text-muted-foreground">
+ Ask TypeSafe Jev typed questions about each observation and get
+ calibrated answers in one call, at a fraction of an LLM judge&apos;s
+ cost. Experimental.
+ </span>
+ </div>
+ <Button type="button" size="sm" onClick={onTry}>
+ Try a decision model
+ </Button>
+ <Button
+ type="button"
+ variant="ghost"
+ size="icon-xs"
+ aria-label="Dismiss"
+ onClick={onDismiss}
+ >
+ <X className="h-3.5 w-3.5" />
+ </Button>
+ </div>
+ );
+}
diff --git a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryMethodBadge/EvaluatorGalleryMethodBadge.tsx b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryMethodBadge/EvaluatorGalleryMethodBadge.tsx
index 60074c07fc..129d111d87 100644
--- a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryMethodBadge/EvaluatorGalleryMethodBadge.tsx
+++ b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryMethodBadge/EvaluatorGalleryMethodBadge.tsx
@@ -3,12 +3,15 @@ import { EvalTemplateTypeEnum, type EvalTemplateType } from "@langfuse/shared";
const METHOD_BADGES = {
[EvalTemplateTypeEnum.CODE]: { Icon: Code2, label: "Code" },
- [EvalTemplateTypeEnum.LLM_AS_JUDGE]: { Icon: Sparkles, label: "LLM judge" },
+ [EvalTemplateTypeEnum.LLM_AS_JUDGE]: {
+ Icon: Sparkles,
+ label: "LLM judge",
+ },
[EvalTemplateTypeEnum.DECISION_MODEL]: {
Icon: Scale,
label: "Decision model",
},
-} as const satisfies Record<EvalTemplateType, unknown>;
+} as const satisfies Record<EvalTemplateType, { Icon: unknown; label: string }>;
export function EvaluatorGalleryMethodBadge({
type,
@@ -18,7 +21,7 @@ export function EvaluatorGalleryMethodBadge({
const { Icon, label } = METHOD_BADGES[type];
return (
- <span className="bg-muted text-muted-foreground inline-flex h-5 shrink-0 items-center gap-1 rounded-full px-2 text-xs leading-none">
+ <span className="bg-muted text-muted-foreground inline-flex h-5 shrink-0 items-center gap-1 rounded-full px-2 text-xs leading-none whitespace-nowrap">
<Icon className="size-3" />
{label}
</span>
diff --git a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/components/EvaluatorTemplateRow/EvaluatorTemplateRow.tsx b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/components/EvaluatorTemplateRow/EvaluatorTemplateRow.tsx
index 37b059d0c7..feede29547 100644
--- a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/components/EvaluatorTemplateRow/EvaluatorTemplateRow.tsx
+++ b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGallerySection/components/EvaluatorTemplateRow/EvaluatorTemplateRow.tsx
@@ -18,7 +18,7 @@ export function EvaluatorTemplateRow({
onClick={() => onSelect(template)}
className="hover:bg-muted/60 @container block w-full cursor-pointer rounded-md px-1 py-2.5 text-left transition-colors"
>
- <span className="grid w-full grid-cols-[minmax(0,1fr)_auto] items-center gap-x-3 gap-y-1 @3xl:grid-cols-[minmax(10rem,18rem)_minmax(0,1fr)_max-content_6rem]">
+ <span className="grid w-full grid-cols-[minmax(0,1fr)_auto] items-center gap-x-3 gap-y-1 @3xl:grid-cols-[minmax(10rem,18rem)_minmax(0,1fr)_max-content_max-content]">
<span
className="col-start-1 row-start-1 min-w-0 truncate text-sm leading-5 font-bold"
title={template.name}
diff --git a/web/src/features/evals/v2/constants/managedTemplatesCatalog.clienttest.ts b/web/src/features/evals/v2/constants/managedTemplatesCatalog.clienttest.ts
index c8e7f9ceec..1827c18b50 100644
--- a/web/src/features/evals/v2/constants/managedTemplatesCatalog.clienttest.ts
+++ b/web/src/features/evals/v2/constants/managedTemplatesCatalog.clienttest.ts
@@ -16,7 +16,7 @@ describe("managed evaluator templates catalog", () => {
({ key }) => key,
);
- expect(MANAGED_TEMPLATES_CATALOG.templates).toHaveLength(20);
+ expect(MANAGED_TEMPLATES_CATALOG.templates).toHaveLength(23);
expect(new Set(templateKeys).size).toBe(templateKeys.length);
for (const template of MANAGED_TEMPLATES_CATALOG.templates) {
diff --git a/web/src/features/evals/v2/constants/managedTemplatesCatalog.ts b/web/src/features/evals/v2/constants/managedTemplatesCatalog.ts
index a046e827f0..42528d3371 100644
--- a/web/src/features/evals/v2/constants/managedTemplatesCatalog.ts
+++ b/web/src/features/evals/v2/constants/managedTemplatesCatalog.ts
@@ -1434,5 +1434,118 @@ Task text: {{task_text}}`,
},
},
},
+ {
+ key: "topic-decision-model",
+ name: "Assign Input Topic",
+ categories: ["classifier", "recommended"],
+ icon: "tags",
+ description:
+ "Assigns the input to one of your topics with a calibrated probability for each option, in one fast decision-model call.",
+ maintainer: "langfuse",
+ evaluator: {
+ type: "DECISION_MODEL",
+ questions: [
+ {
+ id: "topic",
+ type: "choice",
+ scoreName: "topic",
+ instructions:
+ "Which topic best matches the user's primary goal in `input`? Replace the options below with your own taxonomy before use.",
+ options: [
+ {
+ value: "support",
+ description: "Asks for product help or troubleshooting.",
+ },
+ {
+ value: "billing",
+ description:
+ "Asks about invoices, pricing, payments, or subscriptions.",
+ },
+ {
+ value: "technical",
+ description: "Asks technical implementation questions.",
+ },
+ {
+ value: "sales",
+ description:
+ "Asks about purchase, trial, demo, or enterprise fit.",
+ },
+ {
+ value: "feedback",
+ description: "Shares feature feedback or product suggestions.",
+ },
+ {
+ value: "other",
+ description: "Does not reasonably fit any topic above.",
+ },
+ ],
+ },
+ ],
+ state: [{ key: "input", defaultMapping: { field: "input" } }],
+ },
+ },
+ {
+ key: "out-of-scope-decision-model",
+ name: "Flag Out-of-Scope Request",
+ categories: ["conversation"],
+ icon: "shield",
+ description:
+ "Returns the probability that the user's request falls outside what the assistant is meant to handle.",
+ maintainer: "langfuse",
+ evaluator: {
+ type: "DECISION_MODEL",
+ questions: [
+ {
+ id: "out_of_scope",
+ type: "noul",
+ scoreName: "out_of_scope",
+ instructions:
+ "Does `input` ask for something outside the assistant's role or supported scope, judging from how the assistant presents itself in `output`?",
+ criteria: {
+ true: "The request concerns an unrelated task, another product, or a personal favor the assistant is not meant to handle.",
+ false:
+ "The request is about the assistant's product or role, even if phrased vaguely or partially answerable.",
+ },
+ },
+ ],
+ state: [
+ { key: "input", defaultMapping: { field: "input" } },
+ { key: "output", defaultMapping: { field: "output" } },
+ ],
+ },
+ },
+ {
+ key: "frustration-decision-model",
+ name: "Rate Customer Frustration",
+ categories: ["conversation"],
+ icon: "frown",
+ description:
+ "Rates how frustrated the user sounds along levels you describe, from calm to ready to leave.",
+ maintainer: "langfuse",
+ evaluator: {
+ type: "DECISION_MODEL",
+ questions: [
+ {
+ id: "frustration",
+ type: "score",
+ scoreName: "customer_frustration",
+ instructions: "How frustrated is the user in `input`?",
+ levels: [
+ { description: "Calm, neutral, or friendly." },
+ { description: "Mildly annoyed but civil." },
+ {
+ description:
+ "Clearly frustrated: repeating themselves, complaining, or using sharp language.",
+ },
+ {
+ description:
+ "Angry: insulting, threatening to leave, or demanding escalation.",
+ },
+ ],
+ },
+ ],
+ state: [{ key: "input", defaultMapping: { field: "input" } }],
+ },
+ },
],
} satisfies ManagedTemplatesCatalog;
diff --git a/web/src/features/evals/v2/fns/templateGallery/managedEvaluatorTemplateService.ts b/web/src/features/evals/v2/fns/templateGallery/managedEvaluatorTemplateService.ts
index 79b263cf7b..0052f0957f 100644
--- a/web/src/features/evals/v2/fns/templateGallery/managedEvaluatorTemplateService.ts
+++ b/web/src/features/evals/v2/fns/templateGallery/managedEvaluatorTemplateService.ts
@@ -5,10 +5,12 @@ export const managedEvaluatorTemplateService = {
search,
category,
type,
+ includeDecisionModels = false,
}: {
search?: string;
category?: string;
type?: "LLM_AS_JUDGE" | "CODE";
+ includeDecisionModels?: boolean;
} = {}) {
const query = search?.trim().toLowerCase();
@@ -17,6 +19,8 @@ export const managedEvaluatorTemplateService = {
categories: MANAGED_TEMPLATES_CATALOG.categories,
templates: MANAGED_TEMPLATES_CATALOG.templates.filter(
(template) =>
+ (includeDecisionModels ||
+ template.evaluator.type !== "DECISION_MODEL") &&
(!query ||
template.name.toLowerCase().includes(query) ||
template.description.toLowerCase().includes(query)) &&
diff --git a/web/src/features/evals/v2/fns/templateGallery/managedTemplateToEvaluatorSetupDraft.ts b/web/src/features/evals/v2/fns/templateGallery/managedTemplateToEvaluatorSetupDraft.ts
index 7872d77701..6ee2dd62cd 100644
--- a/web/src/features/evals/v2/fns/templateGallery/managedTemplateToEvaluatorSetupDraft.ts
+++ b/web/src/features/evals/v2/fns/templateGallery/managedTemplateToEvaluatorSetupDraft.ts
@@ -3,36 +3,55 @@ import type {
ManagedTemplate,
} from "@/src/features/evals/v2/types/templateGallery";
+function managedTemplateDefinition(
+ evaluator: ManagedTemplate["evaluator"],
+): EvaluatorSetupDraft["definition"] {
+ switch (evaluator.type) {
+ case "LLM_AS_JUDGE":
+ return {
+ type: evaluator.type,
+ promptMessages: evaluator.promptMessages,
+ provider: null,
+ model: null,
+ modelParams: null,
+ vars: evaluator.variables.map(({ name }) => name),
+ variableMapping: evaluator.variables.map(
+ ({ name, defaultMapping }) => ({
+ templateVariable: name,
+ selectedColumnId: defaultMapping.field,
+ jsonSelector: null,
+ }),
+ ),
+ outputDefinition: evaluator.outputDefinition,
+ };
+ case "CODE":
+ return {
+ type: evaluator.type,
+ sourceCode: evaluator.source,
+ sourceCodeLanguage: evaluator.language,
+ };
+ case "DECISION_MODEL":
+ return {
+ type: evaluator.type,
+ questions: evaluator.questions,
+ provider: "",
+ model: "",
+ vars: evaluator.state.map(({ key }) => key),
+ variableMapping: evaluator.state.map(({ key, defaultMapping }) => ({
+ templateVariable: key,
+ selectedColumnId: defaultMapping.field,
+ jsonSelector: null,
+ })),
+ };
+ }
+}
+
export function managedTemplateToEvaluatorSetupDraft(
template: ManagedTemplate,
): EvaluatorSetupDraft {
- const definition =
- template.evaluator.type === "LLM_AS_JUDGE"
- ? {
- type: template.evaluator.type,
- promptMessages: template.evaluator.promptMessages,
- provider: null,
- model: null,
- modelParams: null,
- vars: template.evaluator.variables.map(({ name }) => name),
- variableMapping: template.evaluator.variables.map(
- ({ name, defaultMapping }) => ({
- templateVariable: name,
- selectedColumnId: defaultMapping.field,
- jsonSelector: null,
- }),
- ),
- outputDefinition: template.evaluator.outputDefinition,
- }
- : {
- type: template.evaluator.type,
- sourceCode: template.evaluator.source,
- sourceCodeLanguage: template.evaluator.language,
- };
-
return {
name: template.name,
description: template.description,
- definition,
+ definition: managedTemplateDefinition(template.evaluator),
};
}
diff --git a/web/src/features/evals/v2/fns/templateGallery/prepareEvaluatorEmptyState.clienttest.ts b/web/src/features/evals/v2/fns/templateGallery/prepareEvaluatorEmptyState.clienttest.ts
index fd8fc7fcf4..84c4e07996 100644
--- a/web/src/features/evals/v2/fns/templateGallery/prepareEvaluatorEmptyState.clienttest.ts
+++ b/web/src/features/evals/v2/fns/templateGallery/prepareEvaluatorEmptyState.clienttest.ts
@@ -11,7 +11,9 @@ describe("prepareEvaluatorEmptyState", () => {
expect(emptyState.docsHref).toBe(EVALUATOR_EMPTY_STATE_DOCS_HREF);
expect(emptyState.templateCount).toBe(
- MANAGED_TEMPLATES_CATALOG.templates.length,
+ MANAGED_TEMPLATES_CATALOG.templates.filter(
+ (template) => template.evaluator.type !== "DECISION_MODEL",
+ ).length,
);
expect(
emptyState.startingPoints.map(({ template, action }) => ({
diff --git a/web/src/features/evals/v2/fns/templateGallery/prepareEvaluatorEmptyState.ts b/web/src/features/evals/v2/fns/templateGallery/prepareEvaluatorEmptyState.ts
index 498e91581a..94d173d33e 100644
--- a/web/src/features/evals/v2/fns/templateGallery/prepareEvaluatorEmptyState.ts
+++ b/web/src/features/evals/v2/fns/templateGallery/prepareEvaluatorEmptyState.ts
@@ -1,4 +1,3 @@
-import { MANAGED_TEMPLATES_CATALOG } from "@/src/features/evals/v2/constants/managedTemplatesCatalog";
import {
EVALUATOR_EMPTY_STATE_DOCS_HREF,
EVALUATOR_EMPTY_STATE_STARTING_POINTS,
@@ -32,7 +31,7 @@ export function prepareEvaluatorEmptyState(): EvaluatorEmptyStateModel {
const template = managedEvaluatorTemplateService.get(point.templateKey);
return template ? [toStartingPoint(point, template)] : [];
}),
- templateCount: MANAGED_TEMPLATES_CATALOG.templates.length,
+ templateCount: managedEvaluatorTemplateService.list().templates.length,
docsHref: EVALUATOR_EMPTY_STATE_DOCS_HREF,
};
}
diff --git a/web/src/features/evals/v2/fns/templateGallery/prepareEvaluatorGallery.ts b/web/src/features/evals/v2/fns/templateGallery/prepareEvaluatorGallery.ts
index e125703f60..c50c5fa470 100644
--- a/web/src/features/evals/v2/fns/templateGallery/prepareEvaluatorGallery.ts
+++ b/web/src/features/evals/v2/fns/templateGallery/prepareEvaluatorGallery.ts
@@ -8,11 +8,16 @@ import { managedEvaluatorTemplateService } from "@/src/features/evals/v2/fns/tem
import { EVALUATOR_GALLERY_PROJECT_SECTION_KEY } from "@/src/features/evals/v2/constants/evaluatorGallery";
const RECOMMENDED_TEMPLATE_ORDER = [
+ "topic-decision-model",
"topic-classifier",
"out-of-scope-request",
"quality-criterion",
] as const;
+const RECOMMENDED_TEMPLATE_REPLACEMENTS: Record<string, string> = {
+ "topic-decision-model": "topic-classifier",
+};
+
const RECOMMENDED_TEMPLATE_RANK = new Map<string, number>(
RECOMMENDED_TEMPLATE_ORDER.map((key, index) => [key, index]),
);
@@ -21,10 +26,12 @@ export function prepareEvaluatorGallery({
customTemplates,
customTemplateCount,
search,
+ includeDecisionModels = false,
}: {
customTemplates: CustomEvaluatorTemplate[];
customTemplateCount: number;
search: string;
+ includeDecisionModels?: boolean;
}): {
navigationItems: GalleryNavigationItem[];
sections: GallerySection[];
@@ -37,21 +44,31 @@ export function prepareEvaluatorGallery({
const filteredCustom = customTemplates.filter((template) =>
matches(template.name, template.description),
);
- const managedCatalog = managedEvaluatorTemplateService.list({ search });
+ const managedCatalog = managedEvaluatorTemplateService.list({
+ search,
+ includeDecisionModels,
+ });
const managedByCategory = new Map(
managedCatalog.categories.map((category) => {
const templatesInCategory = managedCatalog.templates.filter((template) =>
template.categories.includes(category.key),
);
+ const replacedKeys = new Set(
+ templatesInCategory.flatMap(
+ (template) => RECOMMENDED_TEMPLATE_REPLACEMENTS[template.key] ?? [],
+ ),
+ );
const orderedTemplates =
category.key === "recommended"
- ? templatesInCategory.toSorted(
- (left, right) =>
- (RECOMMENDED_TEMPLATE_RANK.get(left.key) ??
- Number.MAX_SAFE_INTEGER) -
- (RECOMMENDED_TEMPLATE_RANK.get(right.key) ??
- Number.MAX_SAFE_INTEGER),
- )
+ ? templatesInCategory
+ .filter((template) => !replacedKeys.has(template.key))
+ .toSorted(
+ (left, right) =>
+ (RECOMMENDED_TEMPLATE_RANK.get(left.key) ??
+ Number.MAX_SAFE_INTEGER) -
+ (RECOMMENDED_TEMPLATE_RANK.get(right.key) ??
+ Number.MAX_SAFE_INTEGER),
+ )
: templatesInCategory;
return [
diff --git a/web/src/features/evals/v2/types/templateGallery.ts b/web/src/features/evals/v2/types/templateGallery.ts
index 9321b5d669..2c9839e7f9 100644
--- a/web/src/features/evals/v2/types/templateGallery.ts
+++ b/web/src/features/evals/v2/types/templateGallery.ts
@@ -1,5 +1,6 @@
import type { LucideIcon } from "lucide-react";
import type {
+ DecisionModelQuestions,
EvalTemplateSourceCodeLanguage,
EvalTemplateType,
EvaluatorPromptMessage,
@@ -35,6 +36,11 @@ export type ManagedTemplate = {
type: Extract<EvalTemplateType, "CODE">;
language: EvalTemplateSourceCodeLanguage;
source: string;
+ }
+ | {
+ type: Extract<EvalTemplateType, "DECISION_MODEL">;
+ questions: DecisionModelQuestions;
+ state: Array<{ key: string; defaultMapping: { field: string } }>;
};
};