Skip to content

An AI-powered search engine with a generative UI

Refactor search.ts:

Robert Douglass935a6f1961f1
diff --git a/lib/tools/search.ts b/lib/tools/search.ts
index 5261ef62..4b15df1e 100644
--- a/lib/tools/search.ts
+++ b/lib/tools/search.ts
@@ -1,14 +1,7 @@
import { getSearchSchemaForModel } from '@/lib/schema/search'
-import {
- SearchResultImage,
- SearchResultItem,
- SearchResults,
- SearXNGResponse,
- SearXNGResult
-} from '@/lib/types'
-import { sanitizeUrl } from '@/lib/utils'
+import { SearchResults } from '@/lib/types'
import { tool } from 'ai'
-import Exa from 'exa-js'
+import { DEFAULT_PROVIDER, SearchProviderType, createSearchProvider } from './search/providers'
/**
* Creates a search tool with the appropriate schema for the given model.
@@ -32,12 +25,11 @@ export function createSearchTool(fullModel: string) {
)
const effectiveSearchDepth = search_depth as 'basic' | 'advanced'
- // Tavily API requires a minimum of 5 characters in the query
- const filledQuery =
- query.length < 5 ? query + ' '.repeat(5 - query.length) : query
+ // Use the original query as is - any provider-specific handling will be done in the provider
+ const filledQuery = query
let searchResult: SearchResults
const searchAPI =
- (process.env.SEARCH_API as 'tavily' | 'exa' | 'searxng') || 'tavily'
+ (process.env.SEARCH_API as SearchProviderType) || DEFAULT_PROVIDER
const effectiveSearchDepthForAPI =
searchAPI === 'searxng' &&
@@ -74,11 +66,9 @@ export function createSearchTool(fullModel: string) {
}
searchResult = await response.json()
} else {
- searchResult = await (searchAPI === 'tavily'
- ? tavilySearch
- : searchAPI === 'exa'
- ? exaSearch
- : searxngSearch)(
+ // Use the provider factory to get the appropriate search provider
+ const searchProvider = createSearchProvider(searchAPI)
+ searchResult = await searchProvider.search(
filledQuery,
effectiveMaxResults,
effectiveSearchDepthForAPI,
@@ -126,174 +116,3 @@ export async function search(
}
)
}
-
-async function tavilySearch(
- query: string,
- maxResults: number = 10,
- searchDepth: 'basic' | 'advanced' = 'basic',
- includeDomains: string[] = [],
- excludeDomains: string[] = []
-): Promise<SearchResults> {
- const apiKey = process.env.TAVILY_API_KEY
- if (!apiKey) {
- throw new Error('TAVILY_API_KEY is not set in the environment variables')
- }
- const includeImageDescriptions = true
- const response = await fetch('https://api.tavily.com/search', {
- method: 'POST',
- headers: {
- 'Content-Type': 'application/json'
- },
- body: JSON.stringify({
- api_key: apiKey,
- query,
- max_results: Math.max(maxResults, 5),
- search_depth: searchDepth,
- include_images: true,
- include_image_descriptions: includeImageDescriptions,
- include_answers: true,
- include_domains: includeDomains,
- exclude_domains: excludeDomains
- })
- })
-
- if (!response.ok) {
- throw new Error(
- `Tavily API error: ${response.status} ${response.statusText}`
- )
- }
-
- const data = await response.json()
- const processedImages = includeImageDescriptions
- ? data.images
- .map(({ url, description }: { url: string; description: string }) => ({
- url: sanitizeUrl(url),
- description
- }))
- .filter(
- (
- image: SearchResultImage
- ): image is { url: string; description: string } =>
- typeof image === 'object' &&
- image.description !== undefined &&
- image.description !== ''
- )
- : data.images.map((url: string) => sanitizeUrl(url))
-
- return {
- ...data,
- images: processedImages
- }
-}
-
-async function exaSearch(
- query: string,
- maxResults: number = 10,
- _searchDepth: string,
- includeDomains: string[] = [],
- excludeDomains: string[] = []
-): Promise<SearchResults> {
- const apiKey = process.env.EXA_API_KEY
- if (!apiKey) {
- throw new Error('EXA_API_KEY is not set in the environment variables')
- }
-
- const exa = new Exa(apiKey)
- const exaResults = await exa.searchAndContents(query, {
- highlights: true,
- numResults: maxResults,
- includeDomains,
- excludeDomains
- })
-
- return {
- results: exaResults.results.map((result: any) => ({
- title: result.title,
- url: result.url,
- content: result.highlight || result.text
- })),
- query,
- images: [],
- number_of_results: exaResults.results.length
- }
-}
-
-async function searxngSearch(
- query: string,
- maxResults: number = 10,
- searchDepth: string,
- includeDomains: string[] = [],
- excludeDomains: string[] = []
-): Promise<SearchResults> {
- const apiUrl = process.env.SEARXNG_API_URL
- if (!apiUrl) {
- throw new Error('SEARXNG_API_URL is not set in the environment variables')
- }
-
- try {
- // Construct the URL with query parameters
- const url = new URL(`${apiUrl}/search`)
- url.searchParams.append('q', query)
- url.searchParams.append('format', 'json')
- url.searchParams.append('categories', 'general,images')
-
- // Apply search depth settings
- if (searchDepth === 'advanced') {
- url.searchParams.append('time_range', '')
- url.searchParams.append('safesearch', '0')
- url.searchParams.append('engines', 'google,bing,duckduckgo,wikipedia')
- } else {
- url.searchParams.append('time_range', 'year')
- url.searchParams.append('safesearch', '1')
- url.searchParams.append('engines', 'google,bing')
- }
-
- // Fetch results from SearXNG
- const response = await fetch(url.toString(), {
- method: 'GET',
- headers: {
- Accept: 'application/json'
- }
- })
-
- if (!response.ok) {
- const errorText = await response.text()
- console.error(`SearXNG API error (${response.status}):`, errorText)
- throw new Error(
- `SearXNG API error: ${response.status} ${response.statusText} - ${errorText}`
- )
- }
-
- const data: SearXNGResponse = await response.json()
-
- // Separate general results and image results, and limit to maxResults
- const generalResults = data.results
- .filter(result => !result.img_src)
- .slice(0, maxResults)
- const imageResults = data.results
- .filter(result => result.img_src)
- .slice(0, maxResults)
-
- // Format the results to match the expected SearchResults structure
- return {
- results: generalResults.map(
- (result: SearXNGResult): SearchResultItem => ({
- title: result.title,
- url: result.url,
- content: result.content
- })
- ),
- query: data.query,
- images: imageResults
- .map(result => {
- const imgSrc = result.img_src || ''
- return imgSrc.startsWith('http') ? imgSrc : `${apiUrl}${imgSrc}`
- })
- .filter(Boolean),
- number_of_results: data.number_of_results
- }
- } catch (error) {
- console.error('SearXNG API error:', error)
- throw error
- }
-}
diff --git a/lib/tools/search/providers/base.ts b/lib/tools/search/providers/base.ts
new file mode 100644
index 00000000..e94c7403
--- /dev/null
+++ b/lib/tools/search/providers/base.ts
@@ -0,0 +1,33 @@
+import { SearchResults } from '@/lib/types'
+
+export interface SearchProvider {
+ search(
+ query: string,
+ maxResults: number,
+ searchDepth: 'basic' | 'advanced',
+ includeDomains: string[],
+ excludeDomains: string[]
+ ): Promise<SearchResults>
+}
+
+export abstract class BaseSearchProvider implements SearchProvider {
+ abstract search(
+ query: string,
+ maxResults: number,
+ searchDepth: 'basic' | 'advanced',
+ includeDomains: string[],
+ excludeDomains: string[]
+ ): Promise<SearchResults>
+
+ protected validateApiKey(key: string | undefined, providerName: string): void {
+ if (!key) {
+ throw new Error(`${providerName}_API_KEY is not set in the environment variables`)
+ }
+ }
+
+ protected validateApiUrl(url: string | undefined, providerName: string): void {
+ if (!url) {
+ throw new Error(`${providerName}_API_URL is not set in the environment variables`)
+ }
+ }
+}
\ No newline at end of file
diff --git a/lib/tools/search/providers/exa.ts b/lib/tools/search/providers/exa.ts
new file mode 100644
index 00000000..d0303bdf
--- /dev/null
+++ b/lib/tools/search/providers/exa.ts
@@ -0,0 +1,35 @@
+import { SearchResults } from '@/lib/types'
+import Exa from 'exa-js'
+import { BaseSearchProvider } from './base'
+
+export class ExaSearchProvider extends BaseSearchProvider {
+ async search(
+ query: string,
+ maxResults: number = 10,
+ _searchDepth: 'basic' | 'advanced' = 'basic',
+ includeDomains: string[] = [],
+ excludeDomains: string[] = []
+ ): Promise<SearchResults> {
+ const apiKey = process.env.EXA_API_KEY
+ this.validateApiKey(apiKey, 'EXA')
+
+ const exa = new Exa(apiKey)
+ const exaResults = await exa.searchAndContents(query, {
+ highlights: true,
+ numResults: maxResults,
+ includeDomains,
+ excludeDomains
+ })
+
+ return {
+ results: exaResults.results.map((result: any) => ({
+ title: result.title,
+ url: result.url,
+ content: result.highlight || result.text
+ })),
+ query,
+ images: [],
+ number_of_results: exaResults.results.length
+ }
+ }
+}
\ No newline at end of file
diff --git a/lib/tools/search/providers/index.ts b/lib/tools/search/providers/index.ts
new file mode 100644
index 00000000..c9101fe6
--- /dev/null
+++ b/lib/tools/search/providers/index.ts
@@ -0,0 +1,29 @@
+import { SearchProvider } from './base'
+import { ExaSearchProvider } from './exa'
+import { SearXNGSearchProvider } from './searxng'
+import { TavilySearchProvider } from './tavily'
+
+export type SearchProviderType = 'tavily' | 'exa' | 'searxng'
+export const DEFAULT_PROVIDER: SearchProviderType = 'tavily'
+
+export function createSearchProvider(type?: SearchProviderType): SearchProvider {
+ const providerType = type || (process.env.SEARCH_API as SearchProviderType) || DEFAULT_PROVIDER
+
+ switch (providerType) {
+ case 'tavily':
+ return new TavilySearchProvider()
+ case 'exa':
+ return new ExaSearchProvider()
+ case 'searxng':
+ return new SearXNGSearchProvider()
+ default:
+ // Default to TavilySearchProvider if an unknown provider is specified
+ return new TavilySearchProvider()
+ }
+}
+
+export type { ExaSearchProvider } from './exa'
+export { SearXNGSearchProvider } from './searxng'
+export { TavilySearchProvider } from './tavily'
+export type { SearchProvider }
+
diff --git a/lib/tools/search/providers/searxng.ts b/lib/tools/search/providers/searxng.ts
new file mode 100644
index 00000000..68530d19
--- /dev/null
+++ b/lib/tools/search/providers/searxng.ts
@@ -0,0 +1,87 @@
+import { SearchResultItem, SearchResults, SearXNGResponse, SearXNGResult } from '@/lib/types'
+import { BaseSearchProvider } from './base'
+
+export class SearXNGSearchProvider extends BaseSearchProvider {
+ async search(
+ query: string,
+ maxResults: number = 10,
+ searchDepth: 'basic' | 'advanced' = 'basic',
+ includeDomains: string[] = [],
+ excludeDomains: string[] = []
+ ): Promise<SearchResults> {
+ const apiUrl = process.env.SEARXNG_API_URL
+ this.validateApiUrl(apiUrl, 'SEARXNG')
+
+ try {
+ // Construct the URL with query parameters
+ const url = new URL(`${apiUrl}/search`)
+ url.searchParams.append('q', query)
+ url.searchParams.append('format', 'json')
+ url.searchParams.append('categories', 'general,images')
+
+ // Apply search depth settings
+ if (searchDepth === 'advanced') {
+ url.searchParams.append('time_range', '')
+ url.searchParams.append('safesearch', '0')
+ url.searchParams.append('engines', 'google,bing,duckduckgo,wikipedia')
+ } else {
+ url.searchParams.append('time_range', 'year')
+ url.searchParams.append('safesearch', '1')
+ url.searchParams.append('engines', 'google,bing')
+ }
+
+ // Apply domain filters if provided
+ if (includeDomains.length > 0) {
+ url.searchParams.append('site', includeDomains.join(','))
+ }
+
+ // Fetch results from SearXNG
+ const response = await fetch(url.toString(), {
+ method: 'GET',
+ headers: {
+ Accept: 'application/json'
+ }
+ })
+
+ if (!response.ok) {
+ const errorText = await response.text()
+ console.error(`SearXNG API error (${response.status}):`, errorText)
+ throw new Error(
+ `SearXNG API error: ${response.status} ${response.statusText} - ${errorText}`
+ )
+ }
+
+ const data: SearXNGResponse = await response.json()
+
+ // Separate general results and image results, and limit to maxResults
+ const generalResults = data.results
+ .filter(result => !result.img_src)
+ .slice(0, maxResults)
+ const imageResults = data.results
+ .filter(result => result.img_src)
+ .slice(0, maxResults)
+
+ // Format the results to match the expected SearchResults structure
+ return {
+ results: generalResults.map(
+ (result: SearXNGResult): SearchResultItem => ({
+ title: result.title,
+ url: result.url,
+ content: result.content
+ })
+ ),
+ query: data.query,
+ images: imageResults
+ .map(result => {
+ const imgSrc = result.img_src || ''
+ return imgSrc.startsWith('http') ? imgSrc : `${apiUrl}${imgSrc}`
+ })
+ .filter(Boolean),
+ number_of_results: data.number_of_results
+ }
+ } catch (error) {
+ console.error('SearXNG API error:', error)
+ throw error
+ }
+ }
+}
\ No newline at end of file
diff --git a/lib/tools/search/providers/tavily.ts b/lib/tools/search/providers/tavily.ts
new file mode 100644
index 00000000..3344cdc4
--- /dev/null
+++ b/lib/tools/search/providers/tavily.ts
@@ -0,0 +1,67 @@
+import { SearchResultImage, SearchResults } from '@/lib/types'
+import { sanitizeUrl } from '@/lib/utils'
+import { BaseSearchProvider } from './base'
+
+export class TavilySearchProvider extends BaseSearchProvider {
+ async search(
+ query: string,
+ maxResults: number = 10,
+ searchDepth: 'basic' | 'advanced' = 'basic',
+ includeDomains: string[] = [],
+ excludeDomains: string[] = []
+ ): Promise<SearchResults> {
+ const apiKey = process.env.TAVILY_API_KEY
+ this.validateApiKey(apiKey, 'TAVILY')
+
+ // Tavily API requires a minimum of 5 characters in the query
+ const filledQuery =
+ query.length < 5 ? query + ' '.repeat(5 - query.length) : query
+
+ const includeImageDescriptions = true
+ const response = await fetch('https://api.tavily.com/search', {
+ method: 'POST',
+ headers: {
+ 'Content-Type': 'application/json'
+ },
+ body: JSON.stringify({
+ api_key: apiKey,
+ query: filledQuery,
+ max_results: Math.max(maxResults, 5),
+ search_depth: searchDepth,
+ include_images: true,
+ include_image_descriptions: includeImageDescriptions,
+ include_answers: true,
+ include_domains: includeDomains,
+ exclude_domains: excludeDomains
+ })
+ })
+
+ if (!response.ok) {
+ throw new Error(
+ `Tavily API error: ${response.status} ${response.statusText}`
+ )
+ }
+
+ const data = await response.json()
+ const processedImages = includeImageDescriptions
+ ? data.images
+ .map(({ url, description }: { url: string; description: string }) => ({
+ url: sanitizeUrl(url),
+ description
+ }))
+ .filter(
+ (
+ image: SearchResultImage
+ ): image is { url: string; description: string } =>
+ typeof image === 'object' &&
+ image.description !== undefined &&
+ image.description !== ''
+ )
+ : data.images.map((url: string) => sanitizeUrl(url))
+
+ return {
+ ...data,
+ images: processedImages
+ }
+ }
+}
\ No newline at end of file