🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(evals): add conversation-signals decision-model gallery template (#17845)
annabellschaee14db3cf652
diff --git a/web/src/features/evals/v2/constants/managedTemplatesCatalog.clienttest.ts b/web/src/features/evals/v2/constants/managedTemplatesCatalog.clienttest.tsindex 1827c18b50..402a3500bf 100644--- a/web/src/features/evals/v2/constants/managedTemplatesCatalog.clienttest.ts+++ b/web/src/features/evals/v2/constants/managedTemplatesCatalog.clienttest.ts@@ -1,4 +1,5 @@import {+ DecisionModelQuestionsSchema,observationVariableMappingList,PersistedEvalOutputDefinitionSchema,} from "@langfuse/shared";@@ -16,7 +17,7 @@ describe("managed evaluator templates catalog", () => {({ key }) => key,);- expect(MANAGED_TEMPLATES_CATALOG.templates).toHaveLength(23);+ expect(MANAGED_TEMPLATES_CATALOG.templates).toHaveLength(24);expect(new Set(templateKeys).size).toBe(templateKeys.length);for (const template of MANAGED_TEMPLATES_CATALOG.templates) {@@ -43,6 +44,23 @@ describe("managed evaluator templates catalog", () => {: [],);}++ if (draft.definition.type === "DECISION_MODEL") {+ expect(+ DecisionModelQuestionsSchema.safeParse(draft.definition.questions)+ .success,+ ).toBe(true);+ expect(+ observationVariableMappingList.safeParse(+ draft.definition.variableMapping,+ ).success,+ ).toBe(true);+ expect(draft.definition.vars).toEqual(+ template.evaluator.type === "DECISION_MODEL"+ ? template.evaluator.state.map(({ key }) => key)+ : [],+ );+ }}});diff --git a/web/src/features/evals/v2/constants/managedTemplatesCatalog.ts b/web/src/features/evals/v2/constants/managedTemplatesCatalog.tsindex 42528d3371..9d72ea1694 100644--- a/web/src/features/evals/v2/constants/managedTemplatesCatalog.ts+++ b/web/src/features/evals/v2/constants/managedTemplatesCatalog.ts@@ -1547,5 +1547,103 @@ Task text: {{task_text}}`,state: [{ key: "input", defaultMapping: { field: "input" } }],},},+ {+ key: "conversation-signals",+ name: "Detect Conversation Signals",+ categories: ["conversation"],+ icon: "messages-square",+ description:+ "Flags seven user-side conversation signals—rephrases, corrections, human handoff, retries, quoted errors, frustration, and success—in one decision-model call.",+ maintainer: "langfuse",+ evaluator: {+ type: "DECISION_MODEL",+ questions: [+ {+ id: "user_rephrase_same",+ type: "noul",+ scoreName: "user_rephrase_same",+ instructions:+ "Does the user rephrase the same question again in `input`?",+ criteria: {+ true: "The user restates an earlier question with different wording, without adding a new goal.",+ false:+ "The user asks something new, continues the same turn, or does not rephrase a prior question.",+ },+ },+ {+ id: "user_correct_agent",+ type: "noul",+ scoreName: "user_correct_agent",+ instructions:+ 'Does the user correct the agent in `input` (e.g. "no, I meant the March invoice")?',+ criteria: {+ true: "The user rejects or amends the agent's understanding of their intent, entity, or request.",+ false:+ "The user answers a clarifying question, adds detail, or continues without correcting the agent.",+ },+ },+ {+ id: "user_requests_human",+ type: "noul",+ scoreName: "user_requests_human",+ instructions:+ "Does the user request the hand-off to a human in `input`?",+ criteria: {+ true: "The user asks to speak with a person, agent, or representative.",+ false:+ "The user stays with the assistant or does not ask for a human.",+ },+ },+ {+ id: "user_repeats_request",+ type: "noul",+ scoreName: "user_repeats_request",+ instructions:+ 'Does the user in `input` repeat an instruction the agent already received, or say "try again"?',+ criteria: {+ true: "The user restates a prior instruction or explicitly asks the agent to try again.",+ false:+ "The user makes a new request or continues without repeating an earlier instruction.",+ },+ },+ {+ id: "user_error_quote_back",+ type: "noul",+ scoreName: "user_error_quote_back",+ instructions:+ 'Does the user in `input` quote back an error back they encountered after having received instructions from the agent("you said X, but...")?',+ criteria: {+ true: 'The user pastes or cites an error after following the agent\'s instructions (e.g. "you said X, but...").',+ false:+ "The user does not quote an error, or the error is unrelated to agent instructions.",+ },+ },+ {+ id: "user_express_frustration",+ type: "noul",+ scoreName: "user_express_frustration",+ instructions:+ 'Does the user express frustration in `input` ("this is useless", "you are not listening", using all caps)?',+ criteria: {+ true: "The user shows clear frustration through wording, insults, or aggressive capitalization.",+ false:+ "The user is neutral, mildly annoyed at most, or does not express frustration.",+ },+ },+ {+ id: "user_confirm_success",+ type: "noul",+ scoreName: "user_confirm_success",+ instructions:+ "Does the user confirm success or thanks the agent in `input`?",+ criteria: {+ true: "The user indicates the issue is resolved or thanks the agent.",+ false: "The user does not confirm resolution or thank the agent.",+ },+ },+ ],+ state: [{ key: "input", defaultMapping: { field: "input" } }],+ },+ },],} satisfies ManagedTemplatesCatalog;diff --git a/web/src/features/evals/v2/fns/templateGallery/prepareEvaluatorGallery.clienttest.ts b/web/src/features/evals/v2/fns/templateGallery/prepareEvaluatorGallery.clienttest.tsindex d1f4fb3b01..13026aa44d 100644--- a/web/src/features/evals/v2/fns/templateGallery/prepareEvaluatorGallery.clienttest.ts+++ b/web/src/features/evals/v2/fns/templateGallery/prepareEvaluatorGallery.clienttest.ts@@ -61,6 +61,7 @@ describe("prepareEvaluatorGallery", () => {"user-distress","out-of-scope-decision-model","frustration-decision-model",+ "conversation-signals",]);});