Skip to content

An AI-powered search engine with a generative UI

feat: improve retrieve tool with clickable preview and regular fetch option

Yoshiki Miurae51cc159a7a1
diff --git a/components/retrieve-preview.tsx b/components/retrieve-preview.tsx
new file mode 100644
index 00000000..badb6213
--- /dev/null
+++ b/components/retrieve-preview.tsx
@@ -0,0 +1,122 @@
+import { ExternalLink, Globe } from 'lucide-react'
+
+interface RetrievePreviewProps {
+ url: string
+ title?: string
+ contentLength?: number
+ status: 'fetching' | 'success' | 'error'
+ error?: string
+ fetchType?: string
+}
+
+export function RetrievePreview({
+ url,
+ title,
+ contentLength,
+ status,
+ error,
+ fetchType = 'Retrieve'
+}: RetrievePreviewProps) {
+ const getPageTitle = () => {
+ if (title) return title
+ try {
+ const domain = new URL(url).hostname
+ return domain.replace('www.', '')
+ } catch {
+ return url
+ }
+ }
+
+ const getFavicon = () => {
+ try {
+ const domain = new URL(url).hostname
+ return `https://www.google.com/s2/favicons?domain=${domain}&sz=16`
+ } catch {
+ return null
+ }
+ }
+
+ if (status === 'fetching') {
+ return (
+ <div className="p-3 bg-card border border-border rounded-lg">
+ <div className="flex items-center justify-between gap-2 w-full">
+ <div className="flex items-center gap-2 min-w-0 flex-1">
+ <Globe className="w-4 h-4 text-muted-foreground animate-pulse flex-shrink-0" />
+ <span
+ className="text-sm text-foreground font-medium block truncate min-w-0"
+ title={getPageTitle()}
+ >
+ {getPageTitle()}
+ </span>
+ </div>
+ <span className="text-xs text-muted-foreground animate-pulse whitespace-nowrap">
+ {fetchType}...
+ </span>
+ </div>
+ </div>
+ )
+ }
+
+ if (status === 'error') {
+ return (
+ <div className="p-3 bg-card border border-destructive rounded-lg">
+ <div className="flex items-center gap-2 w-full">
+ <Globe className="w-4 h-4 text-muted-foreground flex-shrink-0" />
+ <span className="text-sm text-destructive block flex-1 truncate min-w-0">
+ Failed to retrieve: {error}
+ </span>
+ </div>
+ </div>
+ )
+ }
+
+ const favicon = getFavicon()
+
+ return (
+ <div
+ className="p-3 bg-card border border-border rounded-lg cursor-pointer hover:border-border/80 transition-colors"
+ onClick={() => window.open(url, '_blank', 'noopener,noreferrer')}
+ >
+ <div className="flex items-center justify-between gap-2 w-full">
+ <div className="flex items-center gap-2 min-w-0 flex-1">
+ {favicon && (
+ // eslint-disable-next-line @next/next/no-img-element
+ <img
+ src={favicon}
+ alt="Favicon"
+ className="w-4 h-4 flex-shrink-0"
+ onError={e => {
+ e.currentTarget.style.display = 'none'
+ e.currentTarget.nextElementSibling?.classList.remove('hidden')
+ }}
+ />
+ )}
+ <Globe
+ className={`w-4 h-4 text-muted-foreground flex-shrink-0 ${
+ favicon ? 'hidden' : ''
+ }`}
+ />
+ <span
+ className="text-sm text-foreground font-medium block truncate min-w-0"
+ title={getPageTitle()}
+ >
+ {getPageTitle()}
+ </span>
+ </div>
+ <div className="flex items-center gap-2 flex-shrink-0 ml-2">
+ {contentLength && contentLength > 0 && (
+ <span className="text-xs text-muted-foreground whitespace-nowrap">
+ {contentLength > 1000
+ ? `${Math.round(contentLength / 1000)}k chars`
+ : `${contentLength} chars`}
+ </span>
+ )}
+ <span className="text-xs text-muted-foreground whitespace-nowrap">
+ {fetchType}
+ </span>
+ <ExternalLink className="w-4 h-4 text-muted-foreground flex-shrink-0" />
+ </div>
+ </div>
+ </div>
+ )
+}
diff --git a/components/retrieve-section.tsx b/components/retrieve-section.tsx
index 8fb7bf17..1b83a191 100644
--- a/components/retrieve-section.tsx
+++ b/components/retrieve-section.tsx
@@ -5,12 +5,7 @@ import { UseChatHelpers } from '@ai-sdk/react'
import { SearchResults as SearchResultsType } from '@/lib/types'
import type { ToolPart, UIDataTypes, UIMessage, UITools } from '@/lib/types/ai'
-import { useArtifact } from '@/components/artifact/artifact-context'
-import { SearchResults } from '@/components/search-results'
-import { Section, ToolArgsSection } from '@/components/section'
-
-import { CollapsibleMessage } from './collapsible-message'
-import { DefaultSkeleton } from './default-skeleton'
+import { RetrievePreview } from './retrieve-preview'
interface RetrieveSectionProps {
tool: ToolPart<'retrieve'>
@@ -19,52 +14,50 @@ interface RetrieveSectionProps {
status?: UseChatHelpers<UIMessage<unknown, UIDataTypes, UITools>>['status']
}
-export function RetrieveSection({
- tool,
- isOpen,
- onOpenChange,
- status
-}: RetrieveSectionProps) {
- const isToolLoading =
- tool.state === 'input-streaming' || tool.state === 'input-available'
- const isChatLoading = status === 'submitted' || status === 'streaming'
- const isLoading = isToolLoading || isChatLoading
-
- const data: SearchResultsType | undefined =
- tool.state === 'output-available' ? tool.output || undefined : undefined
+export function RetrieveSection({ tool }: RetrieveSectionProps) {
const url = tool.input?.url
-
- const { open } = useArtifact()
- const header = (
- <button
- type="button"
- onClick={() => open(tool)}
- className="flex items-center justify-between w-full text-left rounded-md p-1 -ml-1"
- title="Open details"
- >
- <ToolArgsSection tool="retrieve" number={data?.results?.length}>
- {url}
- </ToolArgsSection>
- </button>
- )
+
+ // Determine the fetch type based on the tool input
+ const fetchType = tool.input?.type === 'api' ? 'API Retrieve' : 'Regular Fetch'
+
+ // Determine the status based on tool output availability
+ let displayStatus: 'fetching' | 'success' | 'error' = 'fetching'
+ let error: string | undefined
+ let title: string | undefined
+ let contentLength: number | undefined
+
+ // Check if output is available
+ if (!tool.output) {
+ // Still fetching
+ displayStatus = 'fetching'
+ } else if (tool.state === 'output-error') {
+ // Error state
+ displayStatus = 'error'
+ error = tool.errorText || 'Failed to retrieve content'
+ } else {
+ // Success state - we have output
+ const data = tool.output as SearchResultsType
+ if (data?.results?.[0]) {
+ displayStatus = 'success'
+ title = data.results[0].title
+ contentLength = data.results[0].content?.length
+ } else {
+ displayStatus = 'error'
+ error = 'No content retrieved'
+ }
+ }
return (
- <CollapsibleMessage
- role="assistant"
- isCollapsible={true}
- header={header}
- isOpen={isOpen}
- onOpenChange={onOpenChange}
- showIcon={false}
- >
- {!isLoading && data ? (
- <Section title="Sources">
- <SearchResults results={data.results} />
- </Section>
- ) : (
- <DefaultSkeleton />
- )}
- </CollapsibleMessage>
+ <div className="w-full">
+ <RetrievePreview
+ url={url || ''}
+ title={title}
+ contentLength={contentLength}
+ status={displayStatus}
+ error={error}
+ fetchType={fetchType}
+ />
+ </div>
)
}
diff --git a/lib/schema/retrieve.tsx b/lib/schema/retrieve.tsx
index b3eb657b..83eca7c0 100644
--- a/lib/schema/retrieve.tsx
+++ b/lib/schema/retrieve.tsx
@@ -2,7 +2,13 @@ import { DeepPartial } from 'ai'
import { z } from 'zod'
export const retrieveSchema = z.object({
- url: z.string().describe('The url to retrieve')
+ url: z.string().describe('The URL to retrieve content from'),
+ type: z
+ .enum(['regular', 'api'])
+ .default('regular')
+ .describe(
+ 'Fetch method: "regular" (default) = fast direct HTML fetch, "api" = advanced extraction for PDFs or complex pages (requires API keys)'
+ )
})
export type PartialInquiry = DeepPartial<typeof retrieveSchema>
diff --git a/lib/tools/retrieve.ts b/lib/tools/retrieve.ts
index 7b8f96e7..9965df1f 100644
--- a/lib/tools/retrieve.ts
+++ b/lib/tools/retrieve.ts
@@ -5,6 +5,88 @@ import { SearchResults as SearchResultsType } from '@/lib/types'
const CONTENT_CHARACTER_LIMIT = 10000
+async function fetchRegularData(
+ url: string
+): Promise<SearchResultsType | null> {
+ try {
+ const controller = new AbortController()
+ const timeoutId = setTimeout(() => controller.abort(), 10000) // 10 second timeout
+
+ const response = await fetch(url, {
+ signal: controller.signal,
+ headers: {
+ 'User-Agent': 'Mozilla/5.0 (compatible; Morphic/1.0)',
+ Accept:
+ 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
+ }
+ })
+
+ clearTimeout(timeoutId)
+
+ if (!response.ok) {
+ throw new Error(`HTTP ${response.status}: ${response.statusText}`)
+ }
+
+ const contentType = response.headers.get('content-type') || ''
+ if (
+ !contentType.includes('text/html') &&
+ !contentType.includes('text/plain')
+ ) {
+ throw new Error(`Unsupported content type: ${contentType}`)
+ }
+
+ const html = await response.text()
+
+ // Extract title
+ const titleMatch = html.match(/<title[^>]*>([^<]*)<\/title>/i)
+ const rawTitle = titleMatch ? titleMatch[1].trim() : new URL(url).hostname
+ const title =
+ rawTitle.length > 100 ? rawTitle.substring(0, 100) + '...' : rawTitle
+
+ // Process HTML content
+ let processedHtml = html
+ .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '') // Remove scripts
+ .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '') // Remove styles
+
+ // Replace img tags with alt text or [IMAGE] markers
+ processedHtml = processedHtml
+ .replace(/<img[^>]+alt\s*=\s*["']([^"']+)["'][^>]*>/gi, ' [IMAGE: $1] ')
+ .replace(/<img[^>]+src\s*=\s*["']([^"']+)["'][^>]*>/gi, ' [IMAGE] ')
+ .replace(/<img[^>]*>/gi, ' [IMAGE] ')
+
+ // Extract text content
+ const textContent = processedHtml
+ .replace(/<[^>]*>/g, ' ') // Remove remaining HTML tags
+ .replace(/\s+/g, ' ') // Normalize whitespace
+ .trim()
+
+ // Limit content length
+ const truncatedContent =
+ textContent.length > CONTENT_CHARACTER_LIMIT
+ ? textContent.substring(0, CONTENT_CHARACTER_LIMIT) + '...[truncated]'
+ : textContent
+
+ return {
+ results: [
+ {
+ title,
+ content: truncatedContent,
+ url
+ }
+ ],
+ query: '',
+ images: []
+ }
+ } catch (error) {
+ if (error instanceof Error && error.name === 'AbortError') {
+ console.error('Request timeout after 10 seconds')
+ return null
+ }
+ console.error('Regular fetch error:', error)
+ return null
+ }
+}
+
async function fetchJinaReaderData(
url: string
): Promise<SearchResultsType | null> {
@@ -78,17 +160,22 @@ async function fetchTavilyExtractData(
}
export const retrieveTool = tool({
- description: 'Retrieve content from the web',
+ description: 'Fetch content from any URL. By default uses "regular" type which performs fast, direct HTML fetching without external APIs - ideal for most websites. Only use "api" type when you need: 1) PDF content extraction, 2) Complex JavaScript-rendered pages, 3) Better markdown formatting, 4) Table extraction. The "api" type requires Jina or Tavily API keys.',
inputSchema: retrieveSchema,
- execute: async ({ url }) => {
+ execute: async ({ url, type = 'regular' }) => {
let results: SearchResultsType | null
- // Use Jina if the API key is set, otherwise use Tavily
- const useJina = process.env.JINA_API_KEY
- if (useJina) {
- results = await fetchJinaReaderData(url)
+ if (type === 'regular') {
+ // Use regular fetch for direct HTML retrieval
+ results = await fetchRegularData(url)
} else {
- results = await fetchTavilyExtractData(url)
+ // Use API-based extraction (Jina or Tavily)
+ const useJina = process.env.JINA_API_KEY
+ if (useJina) {
+ results = await fetchJinaReaderData(url)
+ } else {
+ results = await fetchTavilyExtractData(url)
+ }
}
if (!results) {