Skip to content

An AI-powered search engine with a generative UI

fix(render): restore related-questions spec emission on the current model (#926)

Yoshiki Miuraef05e3605165
diff --git a/lib/render/__tests__/llm-related-questions.e2e.test.ts b/lib/render/__tests__/llm-related-questions.e2e.test.ts
new file mode 100644
index 00000000..419ea3bd
--- /dev/null
+++ b/lib/render/__tests__/llm-related-questions.e2e.test.ts
@@ -0,0 +1,144 @@
+/**
+ * E2E test: verify that an LLM emits a well-formed related questions spec
+ * block for substantive answers while skipping it for trivial answers.
+ *
+ * This test hits a real LLM API and is therefore gated behind the
+ * `RUN_LLM_E2E=1` environment variable. CI runs without the flag and the
+ * test is skipped. To run locally:
+ *
+ * RUN_LLM_E2E=1 OPENAI_API_KEY=sk-... bun run test llm-related-questions.e2e
+ */
+import { generateText, stepCountIs, tool } from 'ai'
+import { describe, expect, test } from 'vitest'
+import { z } from 'zod'
+
+import cloudConfig from '@/config/models/cloud.json'
+
+import { getAdaptiveModePrompt } from '@/lib/agents/prompts/search-mode-prompts'
+import { createSearchTool } from '@/lib/tools/search'
+import { getModel } from '@/lib/utils/registry'
+
+import { parseSpecBlock } from '../parse-spec-block'
+
+const RUN = process.env.RUN_LLM_E2E === '1'
+
+// Mirror the deployed adaptive configuration. Emission is sensitive to the
+// provider options (notably reasoning effort), so a run without them does not
+// exercise what production does.
+const ADAPTIVE = cloudConfig.models.adaptive
+const MODEL =
+ process.env.LLM_E2E_MODEL || `${ADAPTIVE.providerId}:${ADAPTIVE.id}`
+const PROVIDER_OPTIONS = ADAPTIVE.providerOptions
+
+const FIXTURE_RESULTS = [
+ {
+ title: 'Mount Fuji climbing routes',
+ url: 'https://example.com/mount-fuji-routes',
+ // Carries the answer to both queries below. Without the height, the trivial
+ // test could pass because the model could not answer at all, which is a
+ // separate skip case.
+ content:
+ 'Mount Fuji stands 3,776 metres (12,389 ft) tall, making it the highest mountain in Japan. It has four main climbing routes. Yoshida is the most popular and has the most huts. Fujinomiya is the shortest but steepest. Subashiri starts below the tree line and joins Yoshida near the summit. Gotemba is the longest and least crowded. The official climbing season is generally July through early September.'
+ }
+]
+
+function createMockSearchTool() {
+ return tool({
+ description: 'Search the web for information.',
+ inputSchema: z.object({
+ query: z.string(),
+ type: z.enum(['optimized', 'general']).optional(),
+ max_results: z.number().optional()
+ }),
+ async execute({ query }) {
+ return {
+ state: 'complete' as const,
+ query,
+ results: FIXTURE_RESULTS,
+ images: [],
+ number_of_results: FIXTURE_RESULTS.length,
+ toolCallId: 'mock-search-1'
+ }
+ },
+ toModelOutput: createSearchTool(MODEL).toModelOutput as never
+ })
+}
+
+function extractSpecBlocks(markdown: string): string[] {
+ const blocks: string[] = []
+ const regex = /```spec\n([\s\S]*?)```/g
+ let match: RegExpExecArray | null
+ while ((match = regex.exec(markdown)) !== null) {
+ blocks.push(match[1])
+ }
+ return blocks
+}
+
+type SpecElement = {
+ type: string
+ props: Record<string, unknown>
+ on?: {
+ press?: {
+ action?: string
+ params?: Record<string, unknown>
+ }
+ }
+}
+
+function findRelatedBlocks(markdown: string): SpecElement[][] {
+ return extractSpecBlocks(markdown)
+ .map(source => parseSpecBlock(source))
+ .map(spec => Object.values(spec.elements) as SpecElement[])
+ .filter(elements =>
+ elements.some(
+ element =>
+ element.type === 'Heading' && element.props.title === 'Related'
+ )
+ )
+}
+
+async function generateAnswer(prompt: string): Promise<string> {
+ const { text } = await generateText({
+ model: getModel(MODEL),
+ system: getAdaptiveModePrompt(),
+ tools: {
+ search: createMockSearchTool()
+ },
+ stopWhen: stepCountIs(6),
+ providerOptions: PROVIDER_OPTIONS,
+ prompt
+ })
+
+ return text
+}
+
+describe.skipIf(!RUN)('LLM related questions output (E2E)', () => {
+ test('substantive answer includes three related question buttons', async () => {
+ const text = await generateAnswer(
+ 'Compare the main Mount Fuji climbing routes and recommend who each route suits.'
+ )
+
+ console.log('[E2E] Substantive model output:\n', text)
+
+ const relatedBlocks = findRelatedBlocks(text)
+ expect(relatedBlocks.length).toBeGreaterThanOrEqual(1)
+
+ const buttons = relatedBlocks[0].filter(
+ element => element.type === 'Button'
+ )
+ expect(buttons).toHaveLength(3)
+
+ for (const button of buttons) {
+ expect(button.on?.press?.action).toBe('submitQuery')
+ expect(button.props.text).toBe(button.on?.press?.params?.query)
+ }
+ }, 120_000)
+
+ test('trivial answer omits the related questions block', async () => {
+ const text = await generateAnswer('How tall is Mount Fuji?')
+
+ console.log('[E2E] Trivial model output:\n', text)
+
+ expect(findRelatedBlocks(text)).toHaveLength(0)
+ }, 120_000)
+})
diff --git a/lib/render/__tests__/prompt.test.ts b/lib/render/__tests__/prompt.test.ts
index c8eb9f54..bbd88db1 100644
--- a/lib/render/__tests__/prompt.test.ts
+++ b/lib/render/__tests__/prompt.test.ts
@@ -3,12 +3,26 @@ import { describe, expect, test } from 'vitest'
import { getImageSpecPrompt, getRelatedQuestionsSpecPrompt } from '../prompt'
describe('render prompts', () => {
- test('makes related questions conditional instead of mandatory', () => {
+ test('includes related questions by default for substantive answers', () => {
const prompt = getRelatedQuestionsSpecPrompt()
- expect(prompt).toContain('Generate related questions only when')
- expect(prompt).toContain('SKIP the spec block entirely')
- expect(prompt).toContain('When in doubt, skip the related questions')
+ expect(prompt).toContain('expected on every substantive answer')
+ expect(prompt).toContain(
+ 'Include the spec block whenever the answer contains substantive information'
+ )
+ expect(prompt).toContain(
+ 'SKIP the spec block entirely (output nothing) only in these cases'
+ )
+ expect(prompt).toContain('Greetings, small talk, or thanks')
+ expect(prompt).toContain(
+ 'Trivial one-off lookups with no natural next step'
+ )
+ expect(prompt).toContain('Meta/operational replies')
+ expect(prompt).toContain('Cases where you could not answer')
+ expect(prompt).toContain(
+ 'Short answers where suggested next questions would be generic or forced'
+ )
+ expect(prompt).toContain('These skip cases are the exception')
expect(prompt).not.toContain('RELATED QUESTIONS (MANDATORY)')
expect(prompt).not.toContain('MUST generate exactly 3')
expect(prompt).not.toContain('Emit exactly ONE related questions')
diff --git a/lib/render/prompt.ts b/lib/render/prompt.ts
index 095c6242..d6a1bc08 100644
--- a/lib/render/prompt.ts
+++ b/lib/render/prompt.ts
@@ -7,20 +7,17 @@
export function getRelatedQuestionsSpecPrompt(): string {
return `
RELATED QUESTIONS:
-Generate related questions only when they create clear next-step value for the user. If you include them, generate exactly 3 follow-up questions in a \`\`\`spec fenced code block after your conclusion.
+After your conclusion, generate exactly 3 follow-up questions in a \`\`\`spec fenced code block. This is expected on every substantive answer; only the narrow skip cases below are exempt.
-Include the spec block only when ALL of these are true:
-- The answer contains substantive information, analysis, comparison, or recommendations.
-- A curious reader would naturally continue with a deeper, practical, or adjacent question.
-- You can anchor each follow-up to concrete details from THIS answer.
+Include the spec block whenever the answer contains substantive information, analysis, comparison, or recommendations. Anchor each follow-up to concrete details from THIS answer.
-SKIP the spec block entirely (output nothing) when follow-ups add little or no value, e.g.:
+SKIP the spec block entirely (output nothing) only in these cases:
- Greetings, small talk, or thanks ("hi", "thanks", "how are you").
- Trivial one-off lookups with no natural next step (simple math, a single fact, a yes/no).
- Meta/operational replies, acknowledgements, or task-completion notices.
- Cases where you could not answer (refusal, clarification request, or empty result).
- Short answers where suggested next questions would be generic or forced.
-When in doubt, skip the related questions. Do not generate them just to satisfy a format.
+These skip cases are the exception. When the answer is substantive, include the spec block.
When you do include them, write the three questions a genuinely curious reader would tap next — not a checklist of "other aspects". Anchor each to THIS answer (use concrete names, options, or numbers from it), and make the three intents distinct:
- Deepen: go further on the single most interesting or surprising point.