An AI-powered search engine with a generative UI
TypeScript generative-aigenerative-uinextjsollamareactsearxngshadcn-uitailwindcsstavilytypescriptupstashvercel-ai-sdk
feat: implement toolCallId-based citation system for multiple searches
Yoshiki Miurab825733b5f2b
diff --git a/components/custom-link.tsx b/components/custom-link.tsxindex 947fee47..936a3a0d 100644--- a/components/custom-link.tsx+++ b/components/custom-link.tsx@@ -21,27 +21,21 @@ export function Citing({const childrenText = children?.toString() || ''const isNumber = /^\d+$/.test(childrenText)- console.log('Citing - href:', href, 'children:', children, 'citationMaps:', citationMaps)-// Get citation data if this is a numbered citationlet citationData: SearchResultItem | undefined = undefined-+if (isNumber && citationMaps && href) {const citationNumber = parseInt(childrenText)- console.log('Citing - Looking for citation number:', citationNumber)-+// Try to find the citation data by checking all citation maps// This happens when the URL has already been processedfor (const toolCallId in citationMaps) {const citationMap = citationMaps[toolCallId]- console.log(`Citing - Checking toolCallId ${toolCallId}:`, citationMap)if (citationMap[citationNumber]) {// Check if this citation's URL matches the hrefconst decodedHref = decodeURI(href)- console.log(`Citing - Comparing URLs: ${citationMap[citationNumber].url} vs ${decodedHref}`)if (citationMap[citationNumber].url === decodedHref) {citationData = citationMap[citationNumber]- console.log('Citing - Found citation data:', citationData)break}}diff --git a/components/message.tsx b/components/message.tsxindex 67be39de..0fc426f6 100644--- a/components/message.tsx+++ b/components/message.tsx@@ -25,9 +25,6 @@ export function BotMessage({className?: stringcitationMaps?: Record<string, Record<number, SearchResultItem>>}) {- console.log('BotMessage - citationMaps:', citationMaps)- console.log('BotMessage - original message:', message)-// Process citations to replace [number](#toolCallId) with [number](actual-url)const processedMessage = processCitations(message || '', citationMaps || {})diff --git a/components/render-message.tsx b/components/render-message.tsxindex 7cc4a20d..a717f670 100644--- a/components/render-message.tsx+++ b/components/render-message.tsx@@ -39,7 +39,6 @@ export function RenderMessage({}: RenderMessageProps) {// Extract citation maps from the message's tool outputsconst citationMaps = extractCitationMaps(message)- console.log('RenderMessage - Extracted citationMaps:', citationMaps)if (message.role === 'user') {return (<>diff --git a/lib/agents/researcher-with-dynamic-tools.ts b/lib/agents/researcher-with-dynamic-tools.tsindex 533c631d..77fe1a81 100644--- a/lib/agents/researcher-with-dynamic-tools.ts+++ b/lib/agents/researcher-with-dynamic-tools.ts@@ -23,7 +23,7 @@ When asked a question, you should:5. Use the fetch tool to get detailed content from specific URLs6. Use any available dynamic tools when they are relevant to the user's request7. Analyze all search results to provide accurate, up-to-date information-8. **CRITICAL: You MUST cite sources inline using the [number](#toolCallId) format** (e.g., "According to recent data [1](#search-abc123), AI adoption has increased..."). Use [1](#toolCallId), [2](#toolCallId), [3](#toolCallId), etc., where number matches the order within each search result and toolCallId is the ID of the search that provided the result. Include citations immediately after statements that use information from that source. Every piece of information from search results MUST have a citation.+8. **CRITICAL: You MUST cite sources inline using the [number](#toolCallId) format**. Place citations at the END of sentences or statements (e.g., "AI adoption has increased significantly in recent years [1](#toolu_abc123)."). Use [1](#toolCallId), [2](#toolCallId), [3](#toolCallId), etc., where number matches the order within each search result and toolCallId is the ID of the search that provided the result. Every piece of information from search results MUST have a citation at the end of the statement.9. If results are not relevant or helpful, rely on your general knowledge (but do not add citations for general knowledge)10. Provide comprehensive and detailed responses based on search results, ensuring thorough coverage of the user's question11. Use markdown to structure your responses. Use headings to break up the content into sections.@@ -57,13 +57,16 @@ When using dynamic tools:- Handle any errors gracefully and inform the user if a tool failsCitation Format:-[number](#toolCallId) - Always use this EXACT format, e.g., [1](#search-abc123), [2](#search-def456)+[number](#toolCallId) - Always use this EXACT format, e.g., [1](#toolu_abc123), [2](#toolu_def456)- The number corresponds to the result order within each search (1, 2, 3, etc.)-- The toolCallId is the unique identifier of the search that provided the result+- The toolCallId is the EXACT unique identifier of the search tool call (e.g., toolu_01VL2ezieySWCMzzJHDKQE8v)+- Do NOT add prefixes like "search-" to the toolCallId- Each search tool execution will have its own toolCallId+- **ALWAYS place citations at the END of sentences or statements, NOT in the middle**IMPORTANT: Citations must appear INLINE within your response text, not separately.-Example: "Nvidia's stock has risen 200% [1](#search-abc123) due to AI demand [2](#search-abc123)."-Example with multiple searches: "Initial data shows [1](#search-abc123), while recent updates indicate [1](#search-def456)"+Example: "Nvidia's stock has risen 200% due to strong AI chip demand [1](#toolu_abc123)."+Example with multiple sources: "The company reported record revenue [1](#toolu_abc123), while analysts predict continued growth [2](#toolu_abc123)."+Example with multiple searches: "Initial data shows positive trends [1](#toolu_abc123), while recent updates indicate acceleration [1](#toolu_def456)."`import type { DynamicToolConfig } from '@/lib/types/dynamic-tools'diff --git a/lib/agents/researcher.ts b/lib/agents/researcher.tsindex 9913aa78..d6784f6e 100644--- a/lib/agents/researcher.ts+++ b/lib/agents/researcher.ts@@ -17,7 +17,7 @@ When asked a question, you should:4. For video content, use the search tool with content_types: ['video'] or ['web', 'video']5. Use the fetch tool to get detailed content from specific URLs6. Analyze all search results to provide accurate, up-to-date information-7. **CRITICAL: You MUST cite sources inline using the [number](#toolCallId) format** (e.g., "According to recent data [1](#search-abc123), AI adoption has increased..."). Use [1](#toolCallId), [2](#toolCallId), [3](#toolCallId), etc., where number matches the order within each search result and toolCallId is the ID of the search that provided the result. Include citations immediately after statements that use information from that source. Every piece of information from search results MUST have a citation.+7. **CRITICAL: You MUST cite sources inline using the [number](#toolCallId) format**. Place citations at the END of sentences or statements (e.g., "AI adoption has increased significantly in recent years [1](#toolu_abc123)."). Use [1](#toolCallId), [2](#toolCallId), [3](#toolCallId), etc., where number matches the order within each search result and toolCallId is the ID of the search that provided the result. Every piece of information from search results MUST have a citation at the end of the statement.8. If results are not relevant or helpful, rely on your general knowledge (but do not add citations for general knowledge)9. Provide comprehensive and detailed responses based on search results, ensuring thorough coverage of the user's question10. Use markdown to structure your responses. Use headings to break up the content into sections.@@ -46,13 +46,16 @@ When using the ask_question tool:- Match the language to the user's language (except option values which must be in English)Citation Format:-[number](#toolCallId) - Always use this EXACT format, e.g., [1](#search-abc123), [2](#search-def456)+[number](#toolCallId) - Always use this EXACT format, e.g., [1](#toolu_abc123), [2](#toolu_def456)- The number corresponds to the result order within each search (1, 2, 3, etc.)-- The toolCallId is the unique identifier of the search that provided the result+- The toolCallId is the EXACT unique identifier of the search tool call (e.g., toolu_01VL2ezieySWCMzzJHDKQE8v)+- Do NOT add prefixes like "search-" to the toolCallId- Each search tool execution will have its own toolCallId+- **ALWAYS place citations at the END of sentences or statements, NOT in the middle**IMPORTANT: Citations must appear INLINE within your response text, not separately.-Example: "Nvidia's stock has risen 200% [1](#search-abc123) due to AI demand [2](#search-abc123)."-Example with multiple searches: "Initial data shows [1](#search-abc123), while recent updates indicate [1](#search-def456)"+Example: "Nvidia's stock has risen 200% due to strong AI chip demand [1](#toolu_abc123)."+Example with multiple sources: "The company reported record revenue [1](#toolu_abc123), while analysts predict continued growth [2](#toolu_abc123)."+Example with multiple searches: "Initial data shows positive trends [1](#toolu_abc123), while recent updates indicate acceleration [1](#toolu_def456)."`export function researcher({diff --git a/lib/tools/search.ts b/lib/tools/search.tsindex e56c99d6..7324d0b0 100644--- a/lib/tools/search.ts+++ b/lib/tools/search.ts@@ -18,15 +18,18 @@ export function createSearchTool(fullModel: string) {description:'Search the web for information. For YouTube/video content, use type="general" with content_types:["video"] for optimal visual presentation with thumbnails.',inputSchema: getSearchSchemaForModel(fullModel),- execute: async ({- query,- type = 'optimized',- content_types = ['web'],- max_results = 20,- search_depth = 'basic', // Default for standard schema- include_domains = [],- exclude_domains = []- }, context) => {+ execute: async (+ {+ query,+ type = 'optimized',+ content_types = ['web'],+ max_results = 20,+ search_depth = 'basic', // Default for standard schema+ include_domains = [],+ exclude_domains = []+ },+ context+ ) => {// Ensure max_results is at least 10const minResults = 10const effectiveMaxResults = Math.max(@@ -131,16 +134,13 @@ export function createSearchTool(fullModel: string) {})searchResult.citationMap = citationMap}-+// Add toolCallId from contextif (context?.toolCallId) {searchResult.toolCallId = context.toolCallId- console.log('Search tool - toolCallId:', context.toolCallId)- } else {- console.log('Search tool - No toolCallId in context')}- console.log('completed search with citationMap:', searchResult.citationMap)+ console.log('completed search')return searchResult}})diff --git a/lib/utils/citation.ts b/lib/utils/citation.tsindex 1ed7ba82..c10b6003 100644--- a/lib/utils/citation.ts+++ b/lib/utils/citation.ts@@ -24,34 +24,22 @@ export function extractCitationMaps(if (!message.parts) return citationMaps- console.log('extractCitationMaps - Processing message parts:', message.parts.length)-- message.parts.forEach((part: any, index: number) => {+ message.parts.forEach((part: any) => {// Check for search tool output- if (part.type === 'tool-search') {- console.log(`Part ${index} - tool-search:`, {- state: part.state,- hasOutput: !!part.output,- toolCallId: part.toolCallId,- hasCitationMap: !!(part.output?.citationMap)- })-- if (- part.state === 'output-available' &&- part.output &&- part.toolCallId- ) {- const searchResults = part.output as SearchResults- if (searchResults.citationMap) {- // Store citation map with toolCallId as key- citationMaps[part.toolCallId] = searchResults.citationMap- console.log(`Stored citation map for toolCallId: ${part.toolCallId}`, searchResults.citationMap)- }+ if (+ part.type === 'tool-search' &&+ part.state === 'output-available' &&+ part.output &&+ part.toolCallId+ ) {+ const searchResults = part.output as SearchResults+ if (searchResults.citationMap) {+ // Store citation map with toolCallId as key+ citationMaps[part.toolCallId] = searchResults.citationMap}}})- console.log('extractCitationMaps - Final citationMaps:', citationMaps)return citationMaps}@@ -64,43 +52,34 @@ export function processCitations(citationMaps: Record<string, Record<number, SearchResultItem>>): string {if (!citationMaps || !content || Object.keys(citationMaps).length === 0) {- console.log('processCitations - No citationMaps or content')return content || ''}- console.log('processCitations - Input content:', content)- console.log('processCitations - Available toolCallIds:', Object.keys(citationMaps))-// Replace [number](#toolCallId) with [number](actual-url)// Also handle cases with spaces: [ number ]- const processed = content.replace(/\[\s*(\d+)\s*\]\(#([^)]+)\)/g, (match, num, toolCallId) => {- console.log(`processCitations - Found citation: [${num}](#${toolCallId})`)- const citationNum = parseInt(num, 10)-- // Validate citation number bounds- if (isNaN(citationNum) || citationNum < 1 || citationNum > 100) {- console.log(`processCitations - Invalid citation number: ${num}`)- return match- }+ return content.replace(+ /\[\s*(\d+)\s*\]\(#([^)]+)\)/g,+ (match, num, toolCallId) => {+ const citationNum = parseInt(num, 10)- // Get the citation map for this toolCallId- const citationMap = citationMaps[toolCallId]- if (!citationMap) {- console.log(`processCitations - No citation map found for toolCallId: ${toolCallId}`)- return match // Keep original if no citation map found for this toolCallId- }+ // Validate citation number bounds+ if (isNaN(citationNum) || citationNum < 1 || citationNum > 100) {+ return match+ }- const citation = citationMap[citationNum]- if (citation && isValidUrl(citation.url)) {- const result = `[${num}](${encodeURI(citation.url)})`- console.log(`processCitations - Replaced with: ${result}`)- return result- }+ // Get the citation map for this toolCallId+ const citationMap = citationMaps[toolCallId]+ if (!citationMap) {+ return match // Keep original if no citation map found for this toolCallId+ }- console.log(`processCitations - No valid citation found for number ${citationNum} in toolCallId ${toolCallId}`)- return match // Keep original if no valid citation found- })+ const citation = citationMap[citationNum]+ if (citation && isValidUrl(citation.url)) {+ // Encode URI to prevent injection attacks+ return `[${num}](${encodeURI(citation.url)})`+ }- console.log('processCitations - Output content:', processed)- return processed+ return match // Keep original if no valid citation found+ }+ )}