An AI-powered search engine with a generative UI
TypeScript generative-aigenerative-uinextjsollamareactsearxngshadcn-uitailwindcsstavilytypescriptupstashvercel-ai-sdk
feat: Support Tavily extract API
Yoshiki Miura08bf7bc25513
diff --git a/lib/agents/tools/retrieve.tsx b/lib/agents/tools/retrieve.tsxindex f9f63940..3cb55911 100644--- a/lib/agents/tools/retrieve.tsx+++ b/lib/agents/tools/retrieve.tsx@@ -5,50 +5,99 @@ import { DefaultSkeleton } from '@/components/default-skeleton'import { SearchResults as SearchResultsType } from '@/lib/types'import RetrieveSection from '@/components/retrieve-section'+const CONTENT_CHARACTER_LIMIT = 10000++async function fetchJinaReaderData(+ url: string+): Promise<SearchResultsType | null> {+ try {+ const response = await fetch(`https://r.jina.ai/${url}`, {+ method: 'GET',+ headers: {+ Accept: 'application/json',+ 'X-With-Generated-Alt': 'true'+ }+ })+ const json = await response.json()+ if (!json.data || json.data.length === 0) {+ return null+ }++ const content = json.data.content.slice(0, CONTENT_CHARACTER_LIMIT)++ return {+ results: [+ {+ title: json.data.title,+ content,+ url: json.data.url+ }+ ],+ query: '',+ images: []+ }+ } catch (error) {+ console.error('Jina Reader API error:', error)+ return null+ }+}++async function fetchTavilyExtractData(+ url: string+): Promise<SearchResultsType | null> {+ try {+ const apiKey = process.env.TAVILY_API_KEY+ const response = await fetch('https://api.tavily.com/extract', {+ method: 'POST',+ headers: {+ 'Content-Type': 'application/json'+ },+ body: JSON.stringify({ api_key: apiKey, urls: [url] })+ })+ const json = await response.json()+ if (!json.results || json.results.length === 0) {+ return null+ }++ const result = json.results[0]+ const content = result.raw_content.slice(0, CONTENT_CHARACTER_LIMIT)++ return {+ results: [+ {+ title: content.slice(0, 100),+ content,+ url: result.url+ }+ ],+ query: '',+ images: []+ }+ } catch (error) {+ console.error('Tavily Extract API error:', error)+ return null+ }+}+export const retrieveTool = ({ uiStream, fullResponse }: ToolProps) =>tool({description: 'Retrieve content from the web',parameters: retrieveSchema,execute: async ({ url }) => {- let hasError = false// Append the search section- uiStream.append(<DefaultSkeleton />)-- let results: SearchResultsType | undefined- try {- const response = await fetch(`https://r.jina.ai/${url}`, {- method: 'GET',- headers: {- Accept: 'application/json',- 'X-With-Generated-Alt': 'true'- }- })- const json = await response.json()- if (!json.data || json.data.length === 0) {- hasError = true- } else {- // Limit the content to 5000 characters- if (json.data.content.length > 5000) {- json.data.content = json.data.content.slice(0, 5000)- }- results = {- results: [- {- title: json.data.title,- content: json.data.content,- url: json.data.url- }- ],- query: '',- images: []- }- }- } catch (error) {- hasError = true- console.error('Retrieve API error:', error)+ uiStream.update(<DefaultSkeleton />)++ let results: SearchResultsType | null++ // Use Jina if the API key is set, otherwise use Tavily+ const useJina = process.env.JINA_API_KEY+ if (useJina) {+ results = await fetchJinaReaderData(url)+ } else {+ results = await fetchTavilyExtractData(url)}- if (hasError || !results) {+ if (!results) {fullResponse = `An error occurred while retrieving "${url}".`uiStream.update(null)return results