🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(evals): refresh all-caps, exact-match, and keyword-overlap templates (#16515)
annabellscha212be781c6cd
diff --git a/web/src/features/evals/v2/constants/managedTemplatesCatalog.clienttest.ts b/web/src/features/evals/v2/constants/managedTemplatesCatalog.clienttest.tsindex cc94189c1f..c8e7f9ceec 100644--- a/web/src/features/evals/v2/constants/managedTemplatesCatalog.clienttest.ts+++ b/web/src/features/evals/v2/constants/managedTemplatesCatalog.clienttest.ts@@ -66,7 +66,11 @@ describe("managed evaluator templates catalog", () => {const createEvaluator = new Function(`${javascript}\nreturn evaluate;`,) as () => (ctx: { observation: { input: unknown } }) => {- scores: Array<{ value: boolean }>;+ scores: Array<{+ value: boolean;+ comment?: string;+ metadata?: { ratioThreshold?: number };+ }>;};const evaluate = createEvaluator();@@ -82,7 +86,197 @@ describe("managed evaluator templates catalog", () => {for (const input of [messages, { messages }]) {const result = evaluate({ observation: { input } });expect(result.scores[0]?.value).toBe(true);+ expect(result.scores[0]?.comment).toMatch(/uppercase/);+ expect(result.scores[0]?.metadata?.ratioThreshold).toBe(0.7);}++ // 70% threshold allows one lowercase word among mostly uppercase letters+ // ("THIS IS COMPLETELY broken" → 16/22 ≈ 72.7%, which fails the old 80% cut).+ const mixedCase = evaluate({+ observation: {+ input: [{ role: "user", content: "THIS IS COMPLETELY broken" }],+ },+ });+ expect(mixedCase.scores[0]?.value).toBe(true);++ // Short shoutouts stay false (4-letter floor).+ const tooShort = evaluate({+ observation: { input: [{ role: "user", content: "WTF" }] },+ });+ expect(tooShort.scores[0]?.value).toBe(false);+ expect(tooShort.scores[0]?.comment).toMatch(/too short/);+ });++ it("exact-match grades only expected_result and ignores sibling keys", () => {+ const template = MANAGED_TEMPLATES_CATALOG.templates.find(+ ({ key }) => key === "exact-match",+ );++ expect(template?.evaluator.type).toBe("CODE");+ if (!template || template.evaluator.type !== "CODE") {+ throw new Error("Exact-match code evaluator template not found");+ }++ const javascript = template.evaluator.source+ .replace(+ "function evaluate(ctx: EvaluationContext): EvaluationResult",+ "function evaluate(ctx)",+ )+ .replace("(value: unknown): value is Record<string, unknown>", "(value)")+ .replaceAll("(value: unknown): unknown", "(value)")+ .replaceAll("(value: unknown)", "(value)")+ .replaceAll(" as Record<string, unknown>", "");+ const createEvaluator = new Function(+ `${javascript}\nreturn evaluate;`,+ ) as () => (ctx: {+ observation: { output: unknown };+ experiment?: { itemExpectedOutput?: unknown };+ }) => {+ scores: Array<{+ value: boolean;+ comment?: string;+ metadata?: {+ expectedUnwrapped?: boolean;+ outputUnwrapped?: boolean;+ };+ }>;+ };++ const evaluate = createEvaluator();++ const match = evaluate({+ observation: { output: "defer_question" },+ experiment: {+ itemExpectedOutput: {+ expected_result: "defer_question",+ sample_reply: "I cannot give financial advice...",+ keyword_overlap: ["financial advice", "stock picks"],+ },+ },+ });+ expect(match.scores[0]?.value).toBe(true);+ expect(match.scores[0]?.comment).toMatch(/compared expected_result only/);+ expect(match.scores[0]?.metadata?.expectedUnwrapped).toBe(true);+ expect(match.scores[0]?.metadata?.outputUnwrapped).toBe(false);++ const wrappedOutputMatch = evaluate({+ observation: {+ output: {+ expected_result: "defer_question",+ extra: "ignored",+ },+ },+ experiment: {+ itemExpectedOutput: {+ expected_result: "defer_question",+ sample_reply: "ignored",+ },+ },+ });+ expect(wrappedOutputMatch.scores[0]?.value).toBe(true);+ expect(wrappedOutputMatch.scores[0]?.metadata?.outputUnwrapped).toBe(true);++ const mismatch = evaluate({+ observation: { output: "answer_directly" },+ experiment: {+ itemExpectedOutput: {+ expected_result: "defer_question",+ sample_reply: "ignored",+ },+ },+ });+ expect(mismatch.scores[0]?.value).toBe(false);+ expect(mismatch.scores[0]?.comment).toMatch(/mismatch/);++ // Without expected_result, compare the whole expected output (key order ignored).+ const wholeObjectMatch = evaluate({+ observation: { output: { b: 2, a: 1 } },+ experiment: { itemExpectedOutput: { a: 1, b: 2 } },+ });+ expect(wholeObjectMatch.scores[0]?.value).toBe(true);+ expect(wholeObjectMatch.scores[0]?.comment).toMatch(+ /no expected_result key/,+ );+ });++ it("keyword-overlap scores the fraction of keyword_overlap hits", () => {+ const template = MANAGED_TEMPLATES_CATALOG.templates.find(+ ({ key }) => key === "keyword-match",+ );++ expect(template?.evaluator.type).toBe("CODE");+ if (!template || template.evaluator.type !== "CODE") {+ throw new Error("Keyword-overlap code evaluator template not found");+ }++ const javascript = template.evaluator.source+ .replace(+ "function evaluate(ctx: EvaluationContext): EvaluationResult",+ "function evaluate(ctx)",+ )+ .replace("(value: unknown): value is Record<string, unknown>", "(value)")+ .replaceAll("(value: unknown): string", "(value)")+ .replaceAll("(value: unknown)", "(value)")+ .replace(+ '(keyword): keyword is string => typeof keyword === "string"',+ '(keyword) => typeof keyword === "string"',+ )+ .replaceAll(" as Record<string, unknown>", "");+ const createEvaluator = new Function(+ `${javascript}\nreturn evaluate;`,+ ) as () => (ctx: {+ observation: { output: unknown };+ experiment?: { itemExpectedOutput?: unknown };+ }) => {+ scores: Array<{+ name?: string;+ value: number;+ dataType?: string;+ comment?: string;+ metadata?: { missing?: string[]; foundCount?: number };+ }>;+ };++ const evaluate = createEvaluator();++ const partial = evaluate({+ observation: {+ output: "I cannot give financial advice or stock picks.",+ },+ experiment: {+ itemExpectedOutput: {+ keyword_overlap: ["financial advice", "stock picks", "Langfuse"],+ expected_result: "defer_question",+ },+ },+ });+ expect(partial.scores).toHaveLength(1);+ expect(partial.scores[0]?.name).toBe("Keyword overlap");+ expect(partial.scores[0]?.dataType).toBe("NUMERIC");+ expect(partial.scores[0]?.value).toBeCloseTo(2 / 3);+ expect(partial.scores[0]?.comment).toMatch(/missing: Langfuse/);+ expect(partial.scores[0]?.metadata?.foundCount).toBe(2);+ expect(partial.scores[0]?.metadata?.missing).toEqual(["Langfuse"]);++ const allFound = evaluate({+ observation: {+ output: "Langfuse helps with financial advice and stock picks.",+ },+ experiment: {+ itemExpectedOutput: ["financial advice", "stock picks", "Langfuse"],+ },+ });+ expect(allFound.scores[0]?.value).toBe(1);+ expect(allFound.scores[0]?.comment).toMatch(/all 3 keywords found/);++ // No keywords → emit no score so the item does not drag averages down.+ const noKeywords = evaluate({+ observation: { output: "anything" },+ experiment: {+ itemExpectedOutput: { expected_result: "defer_question" },+ },+ });+ expect(noKeywords.scores).toEqual([]);});it("ships code evaluator templates that pass client validation", async () => {diff --git a/web/src/features/evals/v2/constants/managedTemplatesCatalog.ts b/web/src/features/evals/v2/constants/managedTemplatesCatalog.tsindex cafd6cbfc8..2d5e42f2f0 100644--- a/web/src/features/evals/v2/constants/managedTemplatesCatalog.ts+++ b/web/src/features/evals/v2/constants/managedTemplatesCatalog.ts@@ -279,8 +279,84 @@ Last user message: {{last_user_message}}`,evaluator: {type: "CODE",language: "TYPESCRIPT",- source:- 'function evaluate(ctx: EvaluationContext): EvaluationResult {\n const extractText = (value: unknown): string => {\n if (typeof value === "string") return value;\n\n if (Array.isArray(value)) {\n return value.map((item) => extractText(item)).filter(Boolean).join("\\n");\n }\n\n if (value !== null && typeof value === "object") {\n const record = value as Record<string, unknown>;\n if ("content" in record) return extractText(record.content);\n if ("text" in record) return extractText(record.text);\n if ("parts" in record) return extractText(record.parts);\n }\n\n return "";\n };\n\n const input = ctx.observation.input;\n const inputRecord =\n input !== null && typeof input === "object" && !Array.isArray(input)\n ? (input as Record<string, unknown>)\n : null;\n const messages = Array.isArray(input)\n ? input\n : Array.isArray(inputRecord?.messages)\n ? inputRecord.messages\n : [];\n\n let message = messages[messages.length - 1];\n for (let index = messages.length - 1; index >= 0; index -= 1) {\n const candidate = messages[index];\n if (candidate === null || typeof candidate !== "object") continue;\n\n const record = candidate as Record<string, unknown>;\n const role = record.role ?? record.type;\n if (role === "user" || role === "human") {\n message = candidate;\n break;\n }\n }\n\n const text = typeof input === "string" ? input : extractText(message ?? input);\n const letters = text.match(/[A-Za-z]/g) ?? [];\n const uppercaseLetters = text.match(/[A-Z]/g) ?? [];\n const uppercaseRatio = letters.length === 0 ? 0 : uppercaseLetters.length / letters.length;\n const isAllCaps = letters.length >= 4 && uppercaseRatio >= 0.8;\n\n return {\n scores: [\n {\n name: "All CAPS",\n value: isAllCaps,\n dataType: "BOOLEAN",\n },\n ],\n };\n}',+ source: `function evaluate(ctx: EvaluationContext): EvaluationResult {+ /**+ * "All CAPS" — true when the latest user message has >= 4 letters and+ * >= 70% of them are uppercase. Counts ASCII letters only, so digits,+ * punctuation, and emoji don't affect the ratio.+ */+ const extractText = (value: unknown): string => {+ if (typeof value === "string") return value;+ if (Array.isArray(value)) {+ return value.map((item) => extractText(item)).filter(Boolean).join("\\n");+ }+ if (value !== null && typeof value === "object") {+ const record = value as Record<string, unknown>;+ if ("content" in record) return extractText(record.content);+ if ("text" in record) return extractText(record.text);+ if ("parts" in record) return extractText(record.parts);+ }+ return "";+ };++ // Accept all three input shapes: string, message array, { messages: [...] }.+ const input = ctx.observation.input;+ const inputRecord =+ input !== null && typeof input === "object" && !Array.isArray(input)+ ? (input as Record<string, unknown>)+ : null;+ const messages = Array.isArray(input)+ ? input+ : Array.isArray(inputRecord?.messages)+ ? inputRecord.messages+ : [];++ // Default to the last message, then walk backwards for the latest user turn.+ let message = messages[messages.length - 1];+ for (let index = messages.length - 1; index >= 0; index -= 1) {+ const candidate = messages[index];+ if (candidate === null || typeof candidate !== "object") continue;+ const record = candidate as Record<string, unknown>;+ const role = record.role ?? record.type; // some SDKs use \`type\` for role+ if (role === "user" || role === "human") {+ message = candidate;+ break;+ }+ }++ // No messages at all: fall back to the whole input object.+ const text = typeof input === "string" ? input : extractText(message ?? input);++ const letters = text.match(/[A-Za-z]/g) ?? [];+ const uppercaseLetters = text.match(/[A-Z]/g) ?? [];+ const uppercaseRatio = letters.length === 0 ? 0 : uppercaseLetters.length / letters.length;+ // 4-letter floor keeps "OK" and "WTF" out; 0.7 allows one lowercase word.+ const isAllCaps = letters.length >= 4 && uppercaseRatio >= 0.7;++ const reasoning =+ letters.length < 4+ ? \`\${letters.length} letters, too short\`+ : \`\${Math.round(uppercaseRatio * 100)}% uppercase of \${letters.length} letters\`;++ return {+ scores: [+ {+ name: "All CAPS",+ value: isAllCaps,+ dataType: "BOOLEAN",+ comment: reasoning,+ metadata: {+ rule: "all_caps",+ letterCount: letters.length,+ uppercaseCount: uppercaseLetters.length,+ uppercaseRatio,+ minLetters: 4,+ ratioThreshold: 0.7,+ },+ },+ ],+ };+}`,},},{@@ -424,32 +500,70 @@ Expected output: {{expected_output}}`,type: "CODE",language: "TYPESCRIPT",source: `function evaluate(ctx: EvaluationContext): EvaluationResult {- const expected = ctx.experiment?.itemExpectedOutput;- const output = ctx.observation.output;-+ /**+ * "Exact match" on one graded field.+ *+ * Compares only \`expected_result\` from the dataset item's expected output;+ * sibling keys ride along for other evaluators and are ignored here. Falls+ * back to comparing the whole value if the key is absent. Object keys are+ * sorted before comparing, so key order doesn't matter; array order does.+ *+ * Example expected output shape:+ * {+ * "expected_result": "defer_question", // graded+ * "sample_reply": "I cannot give financial advice...", // ignored+ * "keyword_overlap": ["financial advice", "stock picks"] // ignored+ * }+ * \`expected_result\` can be any JSON value — string, number, array, object.+ */+ const RESULT_KEY = "expected_result";++ const isRecord = (value: unknown): value is Record<string, unknown> =>+ value !== null && typeof value === "object" && !Array.isArray(value);++ // Unwrap the graded field when present, otherwise grade the whole value.+ const pick = (value: unknown) =>+ isRecord(value) && RESULT_KEY in value+ ? { value: value[RESULT_KEY], unwrapped: true }+ : { value, unwrapped: false };++ // Sort keys recursively so {a,b} and {b,a} stringify identically.const normalize = (value: unknown): unknown => {- if (Array.isArray(value)) {- return value.map((item) => normalize(item));- }-- if (value !== null && typeof value === "object") {- const record = value as Record<string, unknown>;- return Object.keys(record)+ if (Array.isArray(value)) return value.map((item) => normalize(item));+ if (isRecord(value)) {+ return Object.keys(value).sort().reduce((acc, key) => {- acc[key] = normalize(record[key]);+ acc[key] = normalize(value[key]);return acc;}, {} as Record<string, unknown>);}-return value;};- const valuesMatch = (left: unknown, right: unknown) =>- JSON.stringify(normalize(left)) === JSON.stringify(normalize(right));+ const preview = (value: unknown) => {+ const serialized = JSON.stringify(normalize(value)) ?? "undefined";+ return serialized.length > 80 ? \`\${serialized.slice(0, 77)}...\` : serialized;+ };- const hasExpected = expected !== undefined && expected !== null;- const matches = hasExpected && valuesMatch(output, expected);+ const expectedPick = pick(ctx.experiment?.itemExpectedOutput);+ // The task may return the bare value or an object carrying the same key.+ const actualPick = pick(ctx.observation.output);++ const hasExpected = expectedPick.value !== undefined && expectedPick.value !== null;+ const matches =+ hasExpected &&+ JSON.stringify(normalize(actualPick.value)) === JSON.stringify(normalize(expectedPick.value));++ // Comment records both the verdict and which field was actually compared.+ const scope = expectedPick.unwrapped+ ? \`compared \${RESULT_KEY} only, other expected keys ignored\`+ : \`no \${RESULT_KEY} key, compared the whole expected output\`;+ const verdict = !hasExpected+ ? \`nothing to compare (\${RESULT_KEY} is null or missing)\`+ : matches+ ? \`match: \${preview(expectedPick.value)}\`+ : \`mismatch: expected \${preview(expectedPick.value)}, got \${preview(actualPick.value)}\`;return {scores: [@@ -457,6 +571,13 @@ Expected output: {{expected_output}}`,name: "Exact match",value: matches,dataType: "BOOLEAN",+ comment: \`\${verdict} — \${scope}\`,+ metadata: {+ rule: "exact_match_on_expected_result",+ resultKey: RESULT_KEY,+ expectedUnwrapped: expectedPick.unwrapped,+ outputUnwrapped: actualPick.unwrapped,+ },},],};@@ -465,53 +586,99 @@ Expected output: {{expected_output}}`,},{key: "keyword-match",- name: "Validate Keyword Match",+ name: "Validate Keyword Overlap",categories: ["quality"],icon: "list-checks",description:- "Checks whether required keywords, phrases, or entities appear in the output.",+ "Scores what fraction of expected keywords from keyword_overlap appear in the output.",maintainer: "langfuse",evaluator: {type: "CODE",language: "TYPESCRIPT",source: `function evaluate(ctx: EvaluationContext): EvaluationResult {- const outputText = typeof ctx.observation.output === "string"- ? ctx.observation.output- : JSON.stringify(ctx.observation.output ?? "");-- const expectedRaw = ctx.experiment?.itemExpectedOutput;-- let expectedObject: Record<string, unknown> | null = null;- if (expectedRaw !== undefined && expectedRaw !== null) {- if (typeof expectedRaw === "string") {- try {- const parsed = JSON.parse(expectedRaw);- if (parsed && typeof parsed === "object") {- expectedObject = parsed as Record<string, unknown>;- }- } catch {- expectedObject = null;- }- } else if (typeof expectedRaw === "object") {- expectedObject = expectedRaw as Record<string, unknown>;+ /**+ * "Keyword overlap" — fraction of expected keywords present in the output.+ *+ * Keywords come from \`keyword_overlap\` in the dataset item's expected output+ * (a bare array is also accepted). Matching is case-insensitive substring+ * matching, so multi-word phrases work. Score is 0-1; the comment shows the+ * percentage and names whatever is missing. Emits no score when the item+ * defines no keywords, so those items don't drag the average down.+ *+ * Example expected output shape:+ * {+ * "keyword_overlap": ["financial advice", "stock picks", "Langfuse"], // graded+ * "expected_result": "defer_question" // ignored+ * }+ */+ const KEYWORDS_KEY = "keyword_overlap";++ const isRecord = (value: unknown): value is Record<string, unknown> =>+ value !== null && typeof value === "object" && !Array.isArray(value);++ // Collapse the output into one searchable string, whatever shape it has.+ const flatten = (value: unknown): string => {+ if (value === null || value === undefined) return "";+ if (typeof value === "string") return value;+ if (typeof value === "number" || typeof value === "boolean") return String(value);+ if (Array.isArray(value)) return value.map((item) => flatten(item)).filter(Boolean).join("\\n");+ if (isRecord(value)) {+ if ("content" in value) return flatten(value.content);+ if ("text" in value) return flatten(value.text);+ if ("parts" in value) return flatten(value.parts);+ return Object.values(value).map((item) => flatten(item)).filter(Boolean).join("\\n");}- }-- const expectedKeywords = Array.isArray(expectedObject?.expected_keywords)- ? expectedObject.expected_keywords.filter((keyword): keyword is string => typeof keyword === "string")- : [];+ return "";+ };- const normalizedOutput = outputText.toLowerCase();- const matches = expectedKeywords.length > 0 && expectedKeywords.every((keyword) =>- normalizedOutput.includes(keyword.toLowerCase()),- );+ const expected = ctx.experiment?.itemExpectedOutput;+ const raw =+ isRecord(expected) && KEYWORDS_KEY in expected+ ? expected[KEYWORDS_KEY]+ : Array.isArray(expected)+ ? expected+ : undefined;++ // Keep non-empty strings only, then drop case-insensitive duplicates.+ const keywords = (Array.isArray(raw) ? raw : [])+ .filter((keyword): keyword is string => typeof keyword === "string")+ .map((keyword) => keyword.trim())+ .filter(Boolean)+ .filter(+ (keyword, index, all) =>+ all.findIndex((other) => other.toLowerCase() === keyword.toLowerCase()) === index,+ );++ // Nothing to grade: return no score rather than a misleading 0.+ if (keywords.length === 0) return { scores: [] };++ const haystack = flatten(ctx.observation.output).toLowerCase();+ const found = keywords.filter((keyword) => haystack.includes(keyword.toLowerCase()));+ const missing = keywords.filter((keyword) => !found.includes(keyword));++ const ratio = found.length / keywords.length;+ const percent = Math.round(ratio * 100);+ const label = keywords.length === 1 ? "keyword" : "keywords";+ const comment =+ missing.length === 0+ ? \`\${percent}% — all \${keywords.length} \${label} found\`+ : \`\${percent}% — \${found.length}/\${keywords.length} \${label} found, missing: \${missing.join(", ")}\`;return {scores: [{- name: "Keyword match",- value: matches,- dataType: "BOOLEAN",+ name: "Keyword overlap",+ value: ratio, // 0-1; use \`percent\` here if you'd rather store 0-100+ dataType: "NUMERIC",+ comment,+ metadata: {+ rule: "keyword_overlap",+ keywordsKey: KEYWORDS_KEY,+ keywordCount: keywords.length,+ foundCount: found.length,+ found,+ missing,+ },},],};