Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(evals): refresh all-caps, exact-match, and keyword-overlap templates (#16515)

annabellscha212be781c6cd
diff --git a/web/src/features/evals/v2/constants/managedTemplatesCatalog.clienttest.ts b/web/src/features/evals/v2/constants/managedTemplatesCatalog.clienttest.ts
index cc94189c1f..c8e7f9ceec 100644
--- a/web/src/features/evals/v2/constants/managedTemplatesCatalog.clienttest.ts
+++ b/web/src/features/evals/v2/constants/managedTemplatesCatalog.clienttest.ts
@@ -66,7 +66,11 @@ describe("managed evaluator templates catalog", () => {
const createEvaluator = new Function(
`${javascript}\nreturn evaluate;`,
) as () => (ctx: { observation: { input: unknown } }) => {
- scores: Array<{ value: boolean }>;
+ scores: Array<{
+ value: boolean;
+ comment?: string;
+ metadata?: { ratioThreshold?: number };
+ }>;
};
const evaluate = createEvaluator();
@@ -82,7 +86,197 @@ describe("managed evaluator templates catalog", () => {
for (const input of [messages, { messages }]) {
const result = evaluate({ observation: { input } });
expect(result.scores[0]?.value).toBe(true);
+ expect(result.scores[0]?.comment).toMatch(/uppercase/);
+ expect(result.scores[0]?.metadata?.ratioThreshold).toBe(0.7);
}
+
+ // 70% threshold allows one lowercase word among mostly uppercase letters
+ // ("THIS IS COMPLETELY broken" → 16/22 ≈ 72.7%, which fails the old 80% cut).
+ const mixedCase = evaluate({
+ observation: {
+ input: [{ role: "user", content: "THIS IS COMPLETELY broken" }],
+ },
+ });
+ expect(mixedCase.scores[0]?.value).toBe(true);
+
+ // Short shoutouts stay false (4-letter floor).
+ const tooShort = evaluate({
+ observation: { input: [{ role: "user", content: "WTF" }] },
+ });
+ expect(tooShort.scores[0]?.value).toBe(false);
+ expect(tooShort.scores[0]?.comment).toMatch(/too short/);
+ });
+
+ it("exact-match grades only expected_result and ignores sibling keys", () => {
+ const template = MANAGED_TEMPLATES_CATALOG.templates.find(
+ ({ key }) => key === "exact-match",
+ );
+
+ expect(template?.evaluator.type).toBe("CODE");
+ if (!template || template.evaluator.type !== "CODE") {
+ throw new Error("Exact-match code evaluator template not found");
+ }
+
+ const javascript = template.evaluator.source
+ .replace(
+ "function evaluate(ctx: EvaluationContext): EvaluationResult",
+ "function evaluate(ctx)",
+ )
+ .replace("(value: unknown): value is Record<string, unknown>", "(value)")
+ .replaceAll("(value: unknown): unknown", "(value)")
+ .replaceAll("(value: unknown)", "(value)")
+ .replaceAll(" as Record<string, unknown>", "");
+ const createEvaluator = new Function(
+ `${javascript}\nreturn evaluate;`,
+ ) as () => (ctx: {
+ observation: { output: unknown };
+ experiment?: { itemExpectedOutput?: unknown };
+ }) => {
+ scores: Array<{
+ value: boolean;
+ comment?: string;
+ metadata?: {
+ expectedUnwrapped?: boolean;
+ outputUnwrapped?: boolean;
+ };
+ }>;
+ };
+
+ const evaluate = createEvaluator();
+
+ const match = evaluate({
+ observation: { output: "defer_question" },
+ experiment: {
+ itemExpectedOutput: {
+ expected_result: "defer_question",
+ sample_reply: "I cannot give financial advice...",
+ keyword_overlap: ["financial advice", "stock picks"],
+ },
+ },
+ });
+ expect(match.scores[0]?.value).toBe(true);
+ expect(match.scores[0]?.comment).toMatch(/compared expected_result only/);
+ expect(match.scores[0]?.metadata?.expectedUnwrapped).toBe(true);
+ expect(match.scores[0]?.metadata?.outputUnwrapped).toBe(false);
+
+ const wrappedOutputMatch = evaluate({
+ observation: {
+ output: {
+ expected_result: "defer_question",
+ extra: "ignored",
+ },
+ },
+ experiment: {
+ itemExpectedOutput: {
+ expected_result: "defer_question",
+ sample_reply: "ignored",
+ },
+ },
+ });
+ expect(wrappedOutputMatch.scores[0]?.value).toBe(true);
+ expect(wrappedOutputMatch.scores[0]?.metadata?.outputUnwrapped).toBe(true);
+
+ const mismatch = evaluate({
+ observation: { output: "answer_directly" },
+ experiment: {
+ itemExpectedOutput: {
+ expected_result: "defer_question",
+ sample_reply: "ignored",
+ },
+ },
+ });
+ expect(mismatch.scores[0]?.value).toBe(false);
+ expect(mismatch.scores[0]?.comment).toMatch(/mismatch/);
+
+ // Without expected_result, compare the whole expected output (key order ignored).
+ const wholeObjectMatch = evaluate({
+ observation: { output: { b: 2, a: 1 } },
+ experiment: { itemExpectedOutput: { a: 1, b: 2 } },
+ });
+ expect(wholeObjectMatch.scores[0]?.value).toBe(true);
+ expect(wholeObjectMatch.scores[0]?.comment).toMatch(
+ /no expected_result key/,
+ );
+ });
+
+ it("keyword-overlap scores the fraction of keyword_overlap hits", () => {
+ const template = MANAGED_TEMPLATES_CATALOG.templates.find(
+ ({ key }) => key === "keyword-match",
+ );
+
+ expect(template?.evaluator.type).toBe("CODE");
+ if (!template || template.evaluator.type !== "CODE") {
+ throw new Error("Keyword-overlap code evaluator template not found");
+ }
+
+ const javascript = template.evaluator.source
+ .replace(
+ "function evaluate(ctx: EvaluationContext): EvaluationResult",
+ "function evaluate(ctx)",
+ )
+ .replace("(value: unknown): value is Record<string, unknown>", "(value)")
+ .replaceAll("(value: unknown): string", "(value)")
+ .replaceAll("(value: unknown)", "(value)")
+ .replace(
+ '(keyword): keyword is string => typeof keyword === "string"',
+ '(keyword) => typeof keyword === "string"',
+ )
+ .replaceAll(" as Record<string, unknown>", "");
+ const createEvaluator = new Function(
+ `${javascript}\nreturn evaluate;`,
+ ) as () => (ctx: {
+ observation: { output: unknown };
+ experiment?: { itemExpectedOutput?: unknown };
+ }) => {
+ scores: Array<{
+ name?: string;
+ value: number;
+ dataType?: string;
+ comment?: string;
+ metadata?: { missing?: string[]; foundCount?: number };
+ }>;
+ };
+
+ const evaluate = createEvaluator();
+
+ const partial = evaluate({
+ observation: {
+ output: "I cannot give financial advice or stock picks.",
+ },
+ experiment: {
+ itemExpectedOutput: {
+ keyword_overlap: ["financial advice", "stock picks", "Langfuse"],
+ expected_result: "defer_question",
+ },
+ },
+ });
+ expect(partial.scores).toHaveLength(1);
+ expect(partial.scores[0]?.name).toBe("Keyword overlap");
+ expect(partial.scores[0]?.dataType).toBe("NUMERIC");
+ expect(partial.scores[0]?.value).toBeCloseTo(2 / 3);
+ expect(partial.scores[0]?.comment).toMatch(/missing: Langfuse/);
+ expect(partial.scores[0]?.metadata?.foundCount).toBe(2);
+ expect(partial.scores[0]?.metadata?.missing).toEqual(["Langfuse"]);
+
+ const allFound = evaluate({
+ observation: {
+ output: "Langfuse helps with financial advice and stock picks.",
+ },
+ experiment: {
+ itemExpectedOutput: ["financial advice", "stock picks", "Langfuse"],
+ },
+ });
+ expect(allFound.scores[0]?.value).toBe(1);
+ expect(allFound.scores[0]?.comment).toMatch(/all 3 keywords found/);
+
+ // No keywords → emit no score so the item does not drag averages down.
+ const noKeywords = evaluate({
+ observation: { output: "anything" },
+ experiment: {
+ itemExpectedOutput: { expected_result: "defer_question" },
+ },
+ });
+ expect(noKeywords.scores).toEqual([]);
});
it("ships code evaluator templates that pass client validation", async () => {
diff --git a/web/src/features/evals/v2/constants/managedTemplatesCatalog.ts b/web/src/features/evals/v2/constants/managedTemplatesCatalog.ts
index cafd6cbfc8..2d5e42f2f0 100644
--- a/web/src/features/evals/v2/constants/managedTemplatesCatalog.ts
+++ b/web/src/features/evals/v2/constants/managedTemplatesCatalog.ts
@@ -279,8 +279,84 @@ Last user message: {{last_user_message}}`,
evaluator: {
type: "CODE",
language: "TYPESCRIPT",
- source:
- 'function evaluate(ctx: EvaluationContext): EvaluationResult {\n const extractText = (value: unknown): string => {\n if (typeof value === "string") return value;\n\n if (Array.isArray(value)) {\n return value.map((item) => extractText(item)).filter(Boolean).join("\\n");\n }\n\n if (value !== null && typeof value === "object") {\n const record = value as Record<string, unknown>;\n if ("content" in record) return extractText(record.content);\n if ("text" in record) return extractText(record.text);\n if ("parts" in record) return extractText(record.parts);\n }\n\n return "";\n };\n\n const input = ctx.observation.input;\n const inputRecord =\n input !== null && typeof input === "object" && !Array.isArray(input)\n ? (input as Record<string, unknown>)\n : null;\n const messages = Array.isArray(input)\n ? input\n : Array.isArray(inputRecord?.messages)\n ? inputRecord.messages\n : [];\n\n let message = messages[messages.length - 1];\n for (let index = messages.length - 1; index >= 0; index -= 1) {\n const candidate = messages[index];\n if (candidate === null || typeof candidate !== "object") continue;\n\n const record = candidate as Record<string, unknown>;\n const role = record.role ?? record.type;\n if (role === "user" || role === "human") {\n message = candidate;\n break;\n }\n }\n\n const text = typeof input === "string" ? input : extractText(message ?? input);\n const letters = text.match(/[A-Za-z]/g) ?? [];\n const uppercaseLetters = text.match(/[A-Z]/g) ?? [];\n const uppercaseRatio = letters.length === 0 ? 0 : uppercaseLetters.length / letters.length;\n const isAllCaps = letters.length >= 4 && uppercaseRatio >= 0.8;\n\n return {\n scores: [\n {\n name: "All CAPS",\n value: isAllCaps,\n dataType: "BOOLEAN",\n },\n ],\n };\n}',
+ source: `function evaluate(ctx: EvaluationContext): EvaluationResult {
+ /**
+ * "All CAPS" — true when the latest user message has >= 4 letters and
+ * >= 70% of them are uppercase. Counts ASCII letters only, so digits,
+ * punctuation, and emoji don't affect the ratio.
+ */
+ const extractText = (value: unknown): string => {
+ if (typeof value === "string") return value;
+ if (Array.isArray(value)) {
+ return value.map((item) => extractText(item)).filter(Boolean).join("\\n");
+ }
+ if (value !== null && typeof value === "object") {
+ const record = value as Record<string, unknown>;
+ if ("content" in record) return extractText(record.content);
+ if ("text" in record) return extractText(record.text);
+ if ("parts" in record) return extractText(record.parts);
+ }
+ return "";
+ };
+
+ // Accept all three input shapes: string, message array, { messages: [...] }.
+ const input = ctx.observation.input;
+ const inputRecord =
+ input !== null && typeof input === "object" && !Array.isArray(input)
+ ? (input as Record<string, unknown>)
+ : null;
+ const messages = Array.isArray(input)
+ ? input
+ : Array.isArray(inputRecord?.messages)
+ ? inputRecord.messages
+ : [];
+
+ // Default to the last message, then walk backwards for the latest user turn.
+ let message = messages[messages.length - 1];
+ for (let index = messages.length - 1; index >= 0; index -= 1) {
+ const candidate = messages[index];
+ if (candidate === null || typeof candidate !== "object") continue;
+ const record = candidate as Record<string, unknown>;
+ const role = record.role ?? record.type; // some SDKs use \`type\` for role
+ if (role === "user" || role === "human") {
+ message = candidate;
+ break;
+ }
+ }
+
+ // No messages at all: fall back to the whole input object.
+ const text = typeof input === "string" ? input : extractText(message ?? input);
+
+ const letters = text.match(/[A-Za-z]/g) ?? [];
+ const uppercaseLetters = text.match(/[A-Z]/g) ?? [];
+ const uppercaseRatio = letters.length === 0 ? 0 : uppercaseLetters.length / letters.length;
+ // 4-letter floor keeps "OK" and "WTF" out; 0.7 allows one lowercase word.
+ const isAllCaps = letters.length >= 4 && uppercaseRatio >= 0.7;
+
+ const reasoning =
+ letters.length < 4
+ ? \`\${letters.length} letters, too short\`
+ : \`\${Math.round(uppercaseRatio * 100)}% uppercase of \${letters.length} letters\`;
+
+ return {
+ scores: [
+ {
+ name: "All CAPS",
+ value: isAllCaps,
+ dataType: "BOOLEAN",
+ comment: reasoning,
+ metadata: {
+ rule: "all_caps",
+ letterCount: letters.length,
+ uppercaseCount: uppercaseLetters.length,
+ uppercaseRatio,
+ minLetters: 4,
+ ratioThreshold: 0.7,
+ },
+ },
+ ],
+ };
+}`,
},
},
{
@@ -424,32 +500,70 @@ Expected output: {{expected_output}}`,
type: "CODE",
language: "TYPESCRIPT",
source: `function evaluate(ctx: EvaluationContext): EvaluationResult {
- const expected = ctx.experiment?.itemExpectedOutput;
- const output = ctx.observation.output;
-
+ /**
+ * "Exact match" on one graded field.
+ *
+ * Compares only \`expected_result\` from the dataset item's expected output;
+ * sibling keys ride along for other evaluators and are ignored here. Falls
+ * back to comparing the whole value if the key is absent. Object keys are
+ * sorted before comparing, so key order doesn't matter; array order does.
+ *
+ * Example expected output shape:
+ * {
+ * "expected_result": "defer_question", // graded
+ * "sample_reply": "I cannot give financial advice...", // ignored
+ * "keyword_overlap": ["financial advice", "stock picks"] // ignored
+ * }
+ * \`expected_result\` can be any JSON value — string, number, array, object.
+ */
+ const RESULT_KEY = "expected_result";
+
+ const isRecord = (value: unknown): value is Record<string, unknown> =>
+ value !== null && typeof value === "object" && !Array.isArray(value);
+
+ // Unwrap the graded field when present, otherwise grade the whole value.
+ const pick = (value: unknown) =>
+ isRecord(value) && RESULT_KEY in value
+ ? { value: value[RESULT_KEY], unwrapped: true }
+ : { value, unwrapped: false };
+
+ // Sort keys recursively so {a,b} and {b,a} stringify identically.
const normalize = (value: unknown): unknown => {
- if (Array.isArray(value)) {
- return value.map((item) => normalize(item));
- }
-
- if (value !== null && typeof value === "object") {
- const record = value as Record<string, unknown>;
- return Object.keys(record)
+ if (Array.isArray(value)) return value.map((item) => normalize(item));
+ if (isRecord(value)) {
+ return Object.keys(value)
.sort()
.reduce((acc, key) => {
- acc[key] = normalize(record[key]);
+ acc[key] = normalize(value[key]);
return acc;
}, {} as Record<string, unknown>);
}
-
return value;
};
- const valuesMatch = (left: unknown, right: unknown) =>
- JSON.stringify(normalize(left)) === JSON.stringify(normalize(right));
+ const preview = (value: unknown) => {
+ const serialized = JSON.stringify(normalize(value)) ?? "undefined";
+ return serialized.length > 80 ? \`\${serialized.slice(0, 77)}...\` : serialized;
+ };
- const hasExpected = expected !== undefined && expected !== null;
- const matches = hasExpected && valuesMatch(output, expected);
+ const expectedPick = pick(ctx.experiment?.itemExpectedOutput);
+ // The task may return the bare value or an object carrying the same key.
+ const actualPick = pick(ctx.observation.output);
+
+ const hasExpected = expectedPick.value !== undefined && expectedPick.value !== null;
+ const matches =
+ hasExpected &&
+ JSON.stringify(normalize(actualPick.value)) === JSON.stringify(normalize(expectedPick.value));
+
+ // Comment records both the verdict and which field was actually compared.
+ const scope = expectedPick.unwrapped
+ ? \`compared \${RESULT_KEY} only, other expected keys ignored\`
+ : \`no \${RESULT_KEY} key, compared the whole expected output\`;
+ const verdict = !hasExpected
+ ? \`nothing to compare (\${RESULT_KEY} is null or missing)\`
+ : matches
+ ? \`match: \${preview(expectedPick.value)}\`
+ : \`mismatch: expected \${preview(expectedPick.value)}, got \${preview(actualPick.value)}\`;
return {
scores: [
@@ -457,6 +571,13 @@ Expected output: {{expected_output}}`,
name: "Exact match",
value: matches,
dataType: "BOOLEAN",
+ comment: \`\${verdict} — \${scope}\`,
+ metadata: {
+ rule: "exact_match_on_expected_result",
+ resultKey: RESULT_KEY,
+ expectedUnwrapped: expectedPick.unwrapped,
+ outputUnwrapped: actualPick.unwrapped,
+ },
},
],
};
@@ -465,53 +586,99 @@ Expected output: {{expected_output}}`,
},
{
key: "keyword-match",
- name: "Validate Keyword Match",
+ name: "Validate Keyword Overlap",
categories: ["quality"],
icon: "list-checks",
description:
- "Checks whether required keywords, phrases, or entities appear in the output.",
+ "Scores what fraction of expected keywords from keyword_overlap appear in the output.",
maintainer: "langfuse",
evaluator: {
type: "CODE",
language: "TYPESCRIPT",
source: `function evaluate(ctx: EvaluationContext): EvaluationResult {
- const outputText = typeof ctx.observation.output === "string"
- ? ctx.observation.output
- : JSON.stringify(ctx.observation.output ?? "");
-
- const expectedRaw = ctx.experiment?.itemExpectedOutput;
-
- let expectedObject: Record<string, unknown> | null = null;
- if (expectedRaw !== undefined && expectedRaw !== null) {
- if (typeof expectedRaw === "string") {
- try {
- const parsed = JSON.parse(expectedRaw);
- if (parsed && typeof parsed === "object") {
- expectedObject = parsed as Record<string, unknown>;
- }
- } catch {
- expectedObject = null;
- }
- } else if (typeof expectedRaw === "object") {
- expectedObject = expectedRaw as Record<string, unknown>;
+ /**
+ * "Keyword overlap" — fraction of expected keywords present in the output.
+ *
+ * Keywords come from \`keyword_overlap\` in the dataset item's expected output
+ * (a bare array is also accepted). Matching is case-insensitive substring
+ * matching, so multi-word phrases work. Score is 0-1; the comment shows the
+ * percentage and names whatever is missing. Emits no score when the item
+ * defines no keywords, so those items don't drag the average down.
+ *
+ * Example expected output shape:
+ * {
+ * "keyword_overlap": ["financial advice", "stock picks", "Langfuse"], // graded
+ * "expected_result": "defer_question" // ignored
+ * }
+ */
+ const KEYWORDS_KEY = "keyword_overlap";
+
+ const isRecord = (value: unknown): value is Record<string, unknown> =>
+ value !== null && typeof value === "object" && !Array.isArray(value);
+
+ // Collapse the output into one searchable string, whatever shape it has.
+ const flatten = (value: unknown): string => {
+ if (value === null || value === undefined) return "";
+ if (typeof value === "string") return value;
+ if (typeof value === "number" || typeof value === "boolean") return String(value);
+ if (Array.isArray(value)) return value.map((item) => flatten(item)).filter(Boolean).join("\\n");
+ if (isRecord(value)) {
+ if ("content" in value) return flatten(value.content);
+ if ("text" in value) return flatten(value.text);
+ if ("parts" in value) return flatten(value.parts);
+ return Object.values(value).map((item) => flatten(item)).filter(Boolean).join("\\n");
}
- }
-
- const expectedKeywords = Array.isArray(expectedObject?.expected_keywords)
- ? expectedObject.expected_keywords.filter((keyword): keyword is string => typeof keyword === "string")
- : [];
+ return "";
+ };
- const normalizedOutput = outputText.toLowerCase();
- const matches = expectedKeywords.length > 0 && expectedKeywords.every((keyword) =>
- normalizedOutput.includes(keyword.toLowerCase()),
- );
+ const expected = ctx.experiment?.itemExpectedOutput;
+ const raw =
+ isRecord(expected) && KEYWORDS_KEY in expected
+ ? expected[KEYWORDS_KEY]
+ : Array.isArray(expected)
+ ? expected
+ : undefined;
+
+ // Keep non-empty strings only, then drop case-insensitive duplicates.
+ const keywords = (Array.isArray(raw) ? raw : [])
+ .filter((keyword): keyword is string => typeof keyword === "string")
+ .map((keyword) => keyword.trim())
+ .filter(Boolean)
+ .filter(
+ (keyword, index, all) =>
+ all.findIndex((other) => other.toLowerCase() === keyword.toLowerCase()) === index,
+ );
+
+ // Nothing to grade: return no score rather than a misleading 0.
+ if (keywords.length === 0) return { scores: [] };
+
+ const haystack = flatten(ctx.observation.output).toLowerCase();
+ const found = keywords.filter((keyword) => haystack.includes(keyword.toLowerCase()));
+ const missing = keywords.filter((keyword) => !found.includes(keyword));
+
+ const ratio = found.length / keywords.length;
+ const percent = Math.round(ratio * 100);
+ const label = keywords.length === 1 ? "keyword" : "keywords";
+ const comment =
+ missing.length === 0
+ ? \`\${percent}% — all \${keywords.length} \${label} found\`
+ : \`\${percent}% — \${found.length}/\${keywords.length} \${label} found, missing: \${missing.join(", ")}\`;
return {
scores: [
{
- name: "Keyword match",
- value: matches,
- dataType: "BOOLEAN",
+ name: "Keyword overlap",
+ value: ratio, // 0-1; use \`percent\` here if you'd rather store 0-100
+ dataType: "NUMERIC",
+ comment,
+ metadata: {
+ rule: "keyword_overlap",
+ keywordsKey: KEYWORDS_KEY,
+ keywordCount: keywords.length,
+ foundCount: found.length,
+ found,
+ missing,
+ },
},
],
};