Skip to content

An AI-powered search engine with a generative UI

feat(evals): record answer length alongside the item label (#982)

Yoshiki Miurad4aba02a14a5
diff --git a/evals/evaluators/__tests__/related-questions.test.ts b/evals/evaluators/__tests__/related-questions.test.ts
new file mode 100644
index 00000000..06681321
--- /dev/null
+++ b/evals/evaluators/__tests__/related-questions.test.ts
@@ -0,0 +1,127 @@
+import { describe, expect, test } from 'vitest'
+
+import { relatedQuestionsEvaluator } from '../related-questions'
+
+function relatedBlock(
+ questions: Array<{ text: string; query?: string; action?: string }>
+): string {
+ const ids = questions.map((_, index) => `q${index + 1}`)
+ const lines = [
+ '{"op":"add","path":"/root","value":"main"}',
+ `{"op":"add","path":"/elements/main","value":{"type":"Stack","props":{"direction":"vertical","gap":"sm"},"children":["header","questions"]}}`,
+ '{"op":"add","path":"/elements/header","value":{"type":"Heading","props":{"title":"Related","icon":"related"},"children":[]}}',
+ `{"op":"add","path":"/elements/questions","value":{"type":"Stack","props":{"direction":"vertical","gap":"xs"},"children":${JSON.stringify(ids)}}}`,
+ ...questions.map((question, index) => {
+ const value = {
+ type: 'Button',
+ props: {
+ text: question.text,
+ variant: 'link',
+ icon: 'arrow-right'
+ },
+ on: {
+ press: {
+ action: question.action ?? 'submitQuery',
+ params: { query: question.query ?? question.text }
+ }
+ },
+ children: []
+ }
+ return `{"op":"add","path":"/elements/${ids[index]}","value":${JSON.stringify(value)}}`
+ })
+ ]
+
+ return ['```spec', ...lines, '```'].join('\n')
+}
+
+const THREE_QUESTIONS = [
+ { text: 'Which Fuji route suits a first-time climber?' },
+ { text: 'How do hut reservations work in peak season?' },
+ { text: 'What gear is required above the 8th station?' }
+]
+
+const ANSWER = 'Mount Fuji has four main climbing routes.\n\n'
+
+// The prose alone, so both cases below can assert the same value: emitting a
+// block must not lengthen what is recorded.
+const ANSWER_CHARS = 'Mount Fuji has four main climbing routes.'.length
+
+async function evaluate(output: unknown, emitsRelated: boolean) {
+ const result = await relatedQuestionsEvaluator({
+ input: { query: 'How tall is Mount Fuji?', results: [] },
+ output,
+ expectedOutput: { emitsRelated }
+ })
+
+ return Array.isArray(result) ? result : [result]
+}
+
+describe('relatedQuestionsEvaluator', () => {
+ test('records answer length without a related block', async () => {
+ const evaluations = await evaluate(ANSWER, false)
+
+ expect(evaluations).toEqual([
+ {
+ name: 'related_questions_emitted',
+ value: 0,
+ dataType: 'BOOLEAN',
+ comment: 'no related block'
+ },
+ {
+ name: 'related_questions_matches_expectation',
+ value: 1,
+ dataType: 'BOOLEAN',
+ comment: 'expected no block, got none'
+ },
+ {
+ name: 'related_questions_answer_chars',
+ value: ANSWER_CHARS,
+ dataType: 'NUMERIC'
+ }
+ ])
+ })
+
+ test('records answer length and keeps well-formedness with a related block', async () => {
+ const output = ANSWER + relatedBlock(THREE_QUESTIONS)
+ const evaluations = await evaluate(output, true)
+
+ expect(evaluations).toEqual([
+ {
+ name: 'related_questions_emitted',
+ value: 1,
+ dataType: 'BOOLEAN',
+ comment: '3 question(s)'
+ },
+ {
+ name: 'related_questions_matches_expectation',
+ value: 1,
+ dataType: 'BOOLEAN',
+ comment: 'expected a block, got a block'
+ },
+ {
+ name: 'related_questions_answer_chars',
+ value: ANSWER_CHARS,
+ dataType: 'NUMERIC'
+ },
+ {
+ name: 'related_questions_wellformed',
+ value: 1,
+ dataType: 'BOOLEAN',
+ comment: 'ok'
+ }
+ ])
+ })
+
+ test('records zero when the output has no readable answer text', async () => {
+ const evaluations = await evaluate(undefined, false)
+
+ expect(evaluations).toContainEqual({
+ name: 'related_questions_answer_chars',
+ value: 0,
+ dataType: 'NUMERIC'
+ })
+ expect(evaluations.map(evaluation => evaluation.name)).not.toContain(
+ 'related_questions_wellformed'
+ )
+ })
+})
diff --git a/evals/evaluators/related-questions.ts b/evals/evaluators/related-questions.ts
index 37f37e5a..d89cb771 100644
--- a/evals/evaluators/related-questions.ts
+++ b/evals/evaluators/related-questions.ts
@@ -1,5 +1,7 @@
import type { Evaluation, Evaluator, RunEvaluator } from '@langfuse/client'
+import { stripSpecBlocks } from '@/lib/render/strip-spec-blocks'
+
import type {
RelatedQuestionsExpected,
RelatedQuestionsInput,
@@ -61,7 +63,8 @@ export const relatedQuestionsEvaluator: Evaluator<
RelatedQuestionsExpected,
RelatedQuestionsMetadata
> = async ({ output, expectedOutput }) => {
- const analysis = analyzeRelatedQuestions(answerText(output))
+ const answer = answerText(output)
+ const analysis = analyzeRelatedQuestions(answer)
const expected = expectedOutput?.emitsRelated ?? true
const evaluations: Evaluation[] = [
@@ -83,6 +86,16 @@ export const relatedQuestionsEvaluator: Evaluator<
}
]
+ // The item label describes the question, while this records the answer.
+ // They are only comparable when both values live on the same item. The spec
+ // block is stripped first, otherwise emitting one lengthens the answer and
+ // the value moves with the outcome it exists to be compared against.
+ evaluations.push({
+ name: 'related_questions_answer_chars',
+ value: stripSpecBlocks(answer).length,
+ dataType: 'NUMERIC'
+ })
+
// Well-formedness is undefined when nothing was emitted. Scoring it 0 there
// would double-count the emission failure and drag the rate down for a
// reason that has nothing to do with formatting.