An AI-powered search engine with a generative UI
TypeScript generative-aigenerative-uinextjsollamareactsearxngshadcn-uitailwindcsstavilytypescriptupstashvercel-ai-sdk
feat: improve retrieve tool with clickable preview and regular fetch option
Yoshiki Miurae51cc159a7a1
diff --git a/components/retrieve-preview.tsx b/components/retrieve-preview.tsxnew file mode 100644index 00000000..badb6213--- /dev/null+++ b/components/retrieve-preview.tsx@@ -0,0 +1,122 @@+import { ExternalLink, Globe } from 'lucide-react'++interface RetrievePreviewProps {+ url: string+ title?: string+ contentLength?: number+ status: 'fetching' | 'success' | 'error'+ error?: string+ fetchType?: string+}++export function RetrievePreview({+ url,+ title,+ contentLength,+ status,+ error,+ fetchType = 'Retrieve'+}: RetrievePreviewProps) {+ const getPageTitle = () => {+ if (title) return title+ try {+ const domain = new URL(url).hostname+ return domain.replace('www.', '')+ } catch {+ return url+ }+ }++ const getFavicon = () => {+ try {+ const domain = new URL(url).hostname+ return `https://www.google.com/s2/favicons?domain=${domain}&sz=16`+ } catch {+ return null+ }+ }++ if (status === 'fetching') {+ return (+ <div className="p-3 bg-card border border-border rounded-lg">+ <div className="flex items-center justify-between gap-2 w-full">+ <div className="flex items-center gap-2 min-w-0 flex-1">+ <Globe className="w-4 h-4 text-muted-foreground animate-pulse flex-shrink-0" />+ <span+ className="text-sm text-foreground font-medium block truncate min-w-0"+ title={getPageTitle()}+ >+ {getPageTitle()}+ </span>+ </div>+ <span className="text-xs text-muted-foreground animate-pulse whitespace-nowrap">+ {fetchType}...+ </span>+ </div>+ </div>+ )+ }++ if (status === 'error') {+ return (+ <div className="p-3 bg-card border border-destructive rounded-lg">+ <div className="flex items-center gap-2 w-full">+ <Globe className="w-4 h-4 text-muted-foreground flex-shrink-0" />+ <span className="text-sm text-destructive block flex-1 truncate min-w-0">+ Failed to retrieve: {error}+ </span>+ </div>+ </div>+ )+ }++ const favicon = getFavicon()++ return (+ <div+ className="p-3 bg-card border border-border rounded-lg cursor-pointer hover:border-border/80 transition-colors"+ onClick={() => window.open(url, '_blank', 'noopener,noreferrer')}+ >+ <div className="flex items-center justify-between gap-2 w-full">+ <div className="flex items-center gap-2 min-w-0 flex-1">+ {favicon && (+ // eslint-disable-next-line @next/next/no-img-element+ <img+ src={favicon}+ alt="Favicon"+ className="w-4 h-4 flex-shrink-0"+ onError={e => {+ e.currentTarget.style.display = 'none'+ e.currentTarget.nextElementSibling?.classList.remove('hidden')+ }}+ />+ )}+ <Globe+ className={`w-4 h-4 text-muted-foreground flex-shrink-0 ${+ favicon ? 'hidden' : ''+ }`}+ />+ <span+ className="text-sm text-foreground font-medium block truncate min-w-0"+ title={getPageTitle()}+ >+ {getPageTitle()}+ </span>+ </div>+ <div className="flex items-center gap-2 flex-shrink-0 ml-2">+ {contentLength && contentLength > 0 && (+ <span className="text-xs text-muted-foreground whitespace-nowrap">+ {contentLength > 1000+ ? `${Math.round(contentLength / 1000)}k chars`+ : `${contentLength} chars`}+ </span>+ )}+ <span className="text-xs text-muted-foreground whitespace-nowrap">+ {fetchType}+ </span>+ <ExternalLink className="w-4 h-4 text-muted-foreground flex-shrink-0" />+ </div>+ </div>+ </div>+ )+}diff --git a/components/retrieve-section.tsx b/components/retrieve-section.tsxindex 8fb7bf17..1b83a191 100644--- a/components/retrieve-section.tsx+++ b/components/retrieve-section.tsx@@ -5,12 +5,7 @@ import { UseChatHelpers } from '@ai-sdk/react'import { SearchResults as SearchResultsType } from '@/lib/types'import type { ToolPart, UIDataTypes, UIMessage, UITools } from '@/lib/types/ai'-import { useArtifact } from '@/components/artifact/artifact-context'-import { SearchResults } from '@/components/search-results'-import { Section, ToolArgsSection } from '@/components/section'--import { CollapsibleMessage } from './collapsible-message'-import { DefaultSkeleton } from './default-skeleton'+import { RetrievePreview } from './retrieve-preview'interface RetrieveSectionProps {tool: ToolPart<'retrieve'>@@ -19,52 +14,50 @@ interface RetrieveSectionProps {status?: UseChatHelpers<UIMessage<unknown, UIDataTypes, UITools>>['status']}-export function RetrieveSection({- tool,- isOpen,- onOpenChange,- status-}: RetrieveSectionProps) {- const isToolLoading =- tool.state === 'input-streaming' || tool.state === 'input-available'- const isChatLoading = status === 'submitted' || status === 'streaming'- const isLoading = isToolLoading || isChatLoading-- const data: SearchResultsType | undefined =- tool.state === 'output-available' ? tool.output || undefined : undefined+export function RetrieveSection({ tool }: RetrieveSectionProps) {const url = tool.input?.url-- const { open } = useArtifact()- const header = (- <button- type="button"- onClick={() => open(tool)}- className="flex items-center justify-between w-full text-left rounded-md p-1 -ml-1"- title="Open details"- >- <ToolArgsSection tool="retrieve" number={data?.results?.length}>- {url}- </ToolArgsSection>- </button>- )++ // Determine the fetch type based on the tool input+ const fetchType = tool.input?.type === 'api' ? 'API Retrieve' : 'Regular Fetch'++ // Determine the status based on tool output availability+ let displayStatus: 'fetching' | 'success' | 'error' = 'fetching'+ let error: string | undefined+ let title: string | undefined+ let contentLength: number | undefined++ // Check if output is available+ if (!tool.output) {+ // Still fetching+ displayStatus = 'fetching'+ } else if (tool.state === 'output-error') {+ // Error state+ displayStatus = 'error'+ error = tool.errorText || 'Failed to retrieve content'+ } else {+ // Success state - we have output+ const data = tool.output as SearchResultsType+ if (data?.results?.[0]) {+ displayStatus = 'success'+ title = data.results[0].title+ contentLength = data.results[0].content?.length+ } else {+ displayStatus = 'error'+ error = 'No content retrieved'+ }+ }return (- <CollapsibleMessage- role="assistant"- isCollapsible={true}- header={header}- isOpen={isOpen}- onOpenChange={onOpenChange}- showIcon={false}- >- {!isLoading && data ? (- <Section title="Sources">- <SearchResults results={data.results} />- </Section>- ) : (- <DefaultSkeleton />- )}- </CollapsibleMessage>+ <div className="w-full">+ <RetrievePreview+ url={url || ''}+ title={title}+ contentLength={contentLength}+ status={displayStatus}+ error={error}+ fetchType={fetchType}+ />+ </div>)}diff --git a/lib/schema/retrieve.tsx b/lib/schema/retrieve.tsxindex b3eb657b..83eca7c0 100644--- a/lib/schema/retrieve.tsx+++ b/lib/schema/retrieve.tsx@@ -2,7 +2,13 @@ import { DeepPartial } from 'ai'import { z } from 'zod'export const retrieveSchema = z.object({- url: z.string().describe('The url to retrieve')+ url: z.string().describe('The URL to retrieve content from'),+ type: z+ .enum(['regular', 'api'])+ .default('regular')+ .describe(+ 'Fetch method: "regular" (default) = fast direct HTML fetch, "api" = advanced extraction for PDFs or complex pages (requires API keys)'+ )})export type PartialInquiry = DeepPartial<typeof retrieveSchema>diff --git a/lib/tools/retrieve.ts b/lib/tools/retrieve.tsindex 7b8f96e7..9965df1f 100644--- a/lib/tools/retrieve.ts+++ b/lib/tools/retrieve.ts@@ -5,6 +5,88 @@ import { SearchResults as SearchResultsType } from '@/lib/types'const CONTENT_CHARACTER_LIMIT = 10000+async function fetchRegularData(+ url: string+): Promise<SearchResultsType | null> {+ try {+ const controller = new AbortController()+ const timeoutId = setTimeout(() => controller.abort(), 10000) // 10 second timeout++ const response = await fetch(url, {+ signal: controller.signal,+ headers: {+ 'User-Agent': 'Mozilla/5.0 (compatible; Morphic/1.0)',+ Accept:+ 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'+ }+ })++ clearTimeout(timeoutId)++ if (!response.ok) {+ throw new Error(`HTTP ${response.status}: ${response.statusText}`)+ }++ const contentType = response.headers.get('content-type') || ''+ if (+ !contentType.includes('text/html') &&+ !contentType.includes('text/plain')+ ) {+ throw new Error(`Unsupported content type: ${contentType}`)+ }++ const html = await response.text()++ // Extract title+ const titleMatch = html.match(/<title[^>]*>([^<]*)<\/title>/i)+ const rawTitle = titleMatch ? titleMatch[1].trim() : new URL(url).hostname+ const title =+ rawTitle.length > 100 ? rawTitle.substring(0, 100) + '...' : rawTitle++ // Process HTML content+ let processedHtml = html+ .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '') // Remove scripts+ .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '') // Remove styles++ // Replace img tags with alt text or [IMAGE] markers+ processedHtml = processedHtml+ .replace(/<img[^>]+alt\s*=\s*["']([^"']+)["'][^>]*>/gi, ' [IMAGE: $1] ')+ .replace(/<img[^>]+src\s*=\s*["']([^"']+)["'][^>]*>/gi, ' [IMAGE] ')+ .replace(/<img[^>]*>/gi, ' [IMAGE] ')++ // Extract text content+ const textContent = processedHtml+ .replace(/<[^>]*>/g, ' ') // Remove remaining HTML tags+ .replace(/\s+/g, ' ') // Normalize whitespace+ .trim()++ // Limit content length+ const truncatedContent =+ textContent.length > CONTENT_CHARACTER_LIMIT+ ? textContent.substring(0, CONTENT_CHARACTER_LIMIT) + '...[truncated]'+ : textContent++ return {+ results: [+ {+ title,+ content: truncatedContent,+ url+ }+ ],+ query: '',+ images: []+ }+ } catch (error) {+ if (error instanceof Error && error.name === 'AbortError') {+ console.error('Request timeout after 10 seconds')+ return null+ }+ console.error('Regular fetch error:', error)+ return null+ }+}+async function fetchJinaReaderData(url: string): Promise<SearchResultsType | null> {@@ -78,17 +160,22 @@ async function fetchTavilyExtractData(}export const retrieveTool = tool({- description: 'Retrieve content from the web',+ description: 'Fetch content from any URL. By default uses "regular" type which performs fast, direct HTML fetching without external APIs - ideal for most websites. Only use "api" type when you need: 1) PDF content extraction, 2) Complex JavaScript-rendered pages, 3) Better markdown formatting, 4) Table extraction. The "api" type requires Jina or Tavily API keys.',inputSchema: retrieveSchema,- execute: async ({ url }) => {+ execute: async ({ url, type = 'regular' }) => {let results: SearchResultsType | null- // Use Jina if the API key is set, otherwise use Tavily- const useJina = process.env.JINA_API_KEY- if (useJina) {- results = await fetchJinaReaderData(url)+ if (type === 'regular') {+ // Use regular fetch for direct HTML retrieval+ results = await fetchRegularData(url)} else {- results = await fetchTavilyExtractData(url)+ // Use API-based extraction (Jina or Tavily)+ const useJina = process.env.JINA_API_KEY+ if (useJina) {+ results = await fetchJinaReaderData(url)+ } else {+ results = await fetchTavilyExtractData(url)+ }}if (!results) {