An AI-powered search engine with a generative UI
TypeScript generative-aigenerative-uinextjsollamareactsearxngshadcn-uitailwindcsstavilytypescriptupstashvercel-ai-sdk
Refactor search.ts:
Robert Douglass935a6f1961f1
diff --git a/lib/tools/search.ts b/lib/tools/search.tsindex 5261ef62..4b15df1e 100644--- a/lib/tools/search.ts+++ b/lib/tools/search.ts@@ -1,14 +1,7 @@import { getSearchSchemaForModel } from '@/lib/schema/search'-import {- SearchResultImage,- SearchResultItem,- SearchResults,- SearXNGResponse,- SearXNGResult-} from '@/lib/types'-import { sanitizeUrl } from '@/lib/utils'+import { SearchResults } from '@/lib/types'import { tool } from 'ai'-import Exa from 'exa-js'+import { DEFAULT_PROVIDER, SearchProviderType, createSearchProvider } from './search/providers'/*** Creates a search tool with the appropriate schema for the given model.@@ -32,12 +25,11 @@ export function createSearchTool(fullModel: string) {)const effectiveSearchDepth = search_depth as 'basic' | 'advanced'- // Tavily API requires a minimum of 5 characters in the query- const filledQuery =- query.length < 5 ? query + ' '.repeat(5 - query.length) : query+ // Use the original query as is - any provider-specific handling will be done in the provider+ const filledQuery = querylet searchResult: SearchResultsconst searchAPI =- (process.env.SEARCH_API as 'tavily' | 'exa' | 'searxng') || 'tavily'+ (process.env.SEARCH_API as SearchProviderType) || DEFAULT_PROVIDERconst effectiveSearchDepthForAPI =searchAPI === 'searxng' &&@@ -74,11 +66,9 @@ export function createSearchTool(fullModel: string) {}searchResult = await response.json()} else {- searchResult = await (searchAPI === 'tavily'- ? tavilySearch- : searchAPI === 'exa'- ? exaSearch- : searxngSearch)(+ // Use the provider factory to get the appropriate search provider+ const searchProvider = createSearchProvider(searchAPI)+ searchResult = await searchProvider.search(filledQuery,effectiveMaxResults,effectiveSearchDepthForAPI,@@ -126,174 +116,3 @@ export async function search(})}--async function tavilySearch(- query: string,- maxResults: number = 10,- searchDepth: 'basic' | 'advanced' = 'basic',- includeDomains: string[] = [],- excludeDomains: string[] = []-): Promise<SearchResults> {- const apiKey = process.env.TAVILY_API_KEY- if (!apiKey) {- throw new Error('TAVILY_API_KEY is not set in the environment variables')- }- const includeImageDescriptions = true- const response = await fetch('https://api.tavily.com/search', {- method: 'POST',- headers: {- 'Content-Type': 'application/json'- },- body: JSON.stringify({- api_key: apiKey,- query,- max_results: Math.max(maxResults, 5),- search_depth: searchDepth,- include_images: true,- include_image_descriptions: includeImageDescriptions,- include_answers: true,- include_domains: includeDomains,- exclude_domains: excludeDomains- })- })-- if (!response.ok) {- throw new Error(- `Tavily API error: ${response.status} ${response.statusText}`- )- }-- const data = await response.json()- const processedImages = includeImageDescriptions- ? data.images- .map(({ url, description }: { url: string; description: string }) => ({- url: sanitizeUrl(url),- description- }))- .filter(- (- image: SearchResultImage- ): image is { url: string; description: string } =>- typeof image === 'object' &&- image.description !== undefined &&- image.description !== ''- )- : data.images.map((url: string) => sanitizeUrl(url))-- return {- ...data,- images: processedImages- }-}--async function exaSearch(- query: string,- maxResults: number = 10,- _searchDepth: string,- includeDomains: string[] = [],- excludeDomains: string[] = []-): Promise<SearchResults> {- const apiKey = process.env.EXA_API_KEY- if (!apiKey) {- throw new Error('EXA_API_KEY is not set in the environment variables')- }-- const exa = new Exa(apiKey)- const exaResults = await exa.searchAndContents(query, {- highlights: true,- numResults: maxResults,- includeDomains,- excludeDomains- })-- return {- results: exaResults.results.map((result: any) => ({- title: result.title,- url: result.url,- content: result.highlight || result.text- })),- query,- images: [],- number_of_results: exaResults.results.length- }-}--async function searxngSearch(- query: string,- maxResults: number = 10,- searchDepth: string,- includeDomains: string[] = [],- excludeDomains: string[] = []-): Promise<SearchResults> {- const apiUrl = process.env.SEARXNG_API_URL- if (!apiUrl) {- throw new Error('SEARXNG_API_URL is not set in the environment variables')- }-- try {- // Construct the URL with query parameters- const url = new URL(`${apiUrl}/search`)- url.searchParams.append('q', query)- url.searchParams.append('format', 'json')- url.searchParams.append('categories', 'general,images')-- // Apply search depth settings- if (searchDepth === 'advanced') {- url.searchParams.append('time_range', '')- url.searchParams.append('safesearch', '0')- url.searchParams.append('engines', 'google,bing,duckduckgo,wikipedia')- } else {- url.searchParams.append('time_range', 'year')- url.searchParams.append('safesearch', '1')- url.searchParams.append('engines', 'google,bing')- }-- // Fetch results from SearXNG- const response = await fetch(url.toString(), {- method: 'GET',- headers: {- Accept: 'application/json'- }- })-- if (!response.ok) {- const errorText = await response.text()- console.error(`SearXNG API error (${response.status}):`, errorText)- throw new Error(- `SearXNG API error: ${response.status} ${response.statusText} - ${errorText}`- )- }-- const data: SearXNGResponse = await response.json()-- // Separate general results and image results, and limit to maxResults- const generalResults = data.results- .filter(result => !result.img_src)- .slice(0, maxResults)- const imageResults = data.results- .filter(result => result.img_src)- .slice(0, maxResults)-- // Format the results to match the expected SearchResults structure- return {- results: generalResults.map(- (result: SearXNGResult): SearchResultItem => ({- title: result.title,- url: result.url,- content: result.content- })- ),- query: data.query,- images: imageResults- .map(result => {- const imgSrc = result.img_src || ''- return imgSrc.startsWith('http') ? imgSrc : `${apiUrl}${imgSrc}`- })- .filter(Boolean),- number_of_results: data.number_of_results- }- } catch (error) {- console.error('SearXNG API error:', error)- throw error- }-}diff --git a/lib/tools/search/providers/base.ts b/lib/tools/search/providers/base.tsnew file mode 100644index 00000000..e94c7403--- /dev/null+++ b/lib/tools/search/providers/base.ts@@ -0,0 +1,33 @@+import { SearchResults } from '@/lib/types'++export interface SearchProvider {+ search(+ query: string,+ maxResults: number,+ searchDepth: 'basic' | 'advanced',+ includeDomains: string[],+ excludeDomains: string[]+ ): Promise<SearchResults>+}++export abstract class BaseSearchProvider implements SearchProvider {+ abstract search(+ query: string,+ maxResults: number,+ searchDepth: 'basic' | 'advanced',+ includeDomains: string[],+ excludeDomains: string[]+ ): Promise<SearchResults>++ protected validateApiKey(key: string | undefined, providerName: string): void {+ if (!key) {+ throw new Error(`${providerName}_API_KEY is not set in the environment variables`)+ }+ }++ protected validateApiUrl(url: string | undefined, providerName: string): void {+ if (!url) {+ throw new Error(`${providerName}_API_URL is not set in the environment variables`)+ }+ }+}\ No newline at end of filediff --git a/lib/tools/search/providers/exa.ts b/lib/tools/search/providers/exa.tsnew file mode 100644index 00000000..d0303bdf--- /dev/null+++ b/lib/tools/search/providers/exa.ts@@ -0,0 +1,35 @@+import { SearchResults } from '@/lib/types'+import Exa from 'exa-js'+import { BaseSearchProvider } from './base'++export class ExaSearchProvider extends BaseSearchProvider {+ async search(+ query: string,+ maxResults: number = 10,+ _searchDepth: 'basic' | 'advanced' = 'basic',+ includeDomains: string[] = [],+ excludeDomains: string[] = []+ ): Promise<SearchResults> {+ const apiKey = process.env.EXA_API_KEY+ this.validateApiKey(apiKey, 'EXA')++ const exa = new Exa(apiKey)+ const exaResults = await exa.searchAndContents(query, {+ highlights: true,+ numResults: maxResults,+ includeDomains,+ excludeDomains+ })++ return {+ results: exaResults.results.map((result: any) => ({+ title: result.title,+ url: result.url,+ content: result.highlight || result.text+ })),+ query,+ images: [],+ number_of_results: exaResults.results.length+ }+ }+}\ No newline at end of filediff --git a/lib/tools/search/providers/index.ts b/lib/tools/search/providers/index.tsnew file mode 100644index 00000000..c9101fe6--- /dev/null+++ b/lib/tools/search/providers/index.ts@@ -0,0 +1,29 @@+import { SearchProvider } from './base'+import { ExaSearchProvider } from './exa'+import { SearXNGSearchProvider } from './searxng'+import { TavilySearchProvider } from './tavily'++export type SearchProviderType = 'tavily' | 'exa' | 'searxng'+export const DEFAULT_PROVIDER: SearchProviderType = 'tavily'++export function createSearchProvider(type?: SearchProviderType): SearchProvider {+ const providerType = type || (process.env.SEARCH_API as SearchProviderType) || DEFAULT_PROVIDER++ switch (providerType) {+ case 'tavily':+ return new TavilySearchProvider()+ case 'exa':+ return new ExaSearchProvider()+ case 'searxng':+ return new SearXNGSearchProvider()+ default:+ // Default to TavilySearchProvider if an unknown provider is specified+ return new TavilySearchProvider()+ }+}++export type { ExaSearchProvider } from './exa'+export { SearXNGSearchProvider } from './searxng'+export { TavilySearchProvider } from './tavily'+export type { SearchProvider }+diff --git a/lib/tools/search/providers/searxng.ts b/lib/tools/search/providers/searxng.tsnew file mode 100644index 00000000..68530d19--- /dev/null+++ b/lib/tools/search/providers/searxng.ts@@ -0,0 +1,87 @@+import { SearchResultItem, SearchResults, SearXNGResponse, SearXNGResult } from '@/lib/types'+import { BaseSearchProvider } from './base'++export class SearXNGSearchProvider extends BaseSearchProvider {+ async search(+ query: string,+ maxResults: number = 10,+ searchDepth: 'basic' | 'advanced' = 'basic',+ includeDomains: string[] = [],+ excludeDomains: string[] = []+ ): Promise<SearchResults> {+ const apiUrl = process.env.SEARXNG_API_URL+ this.validateApiUrl(apiUrl, 'SEARXNG')++ try {+ // Construct the URL with query parameters+ const url = new URL(`${apiUrl}/search`)+ url.searchParams.append('q', query)+ url.searchParams.append('format', 'json')+ url.searchParams.append('categories', 'general,images')++ // Apply search depth settings+ if (searchDepth === 'advanced') {+ url.searchParams.append('time_range', '')+ url.searchParams.append('safesearch', '0')+ url.searchParams.append('engines', 'google,bing,duckduckgo,wikipedia')+ } else {+ url.searchParams.append('time_range', 'year')+ url.searchParams.append('safesearch', '1')+ url.searchParams.append('engines', 'google,bing')+ }++ // Apply domain filters if provided+ if (includeDomains.length > 0) {+ url.searchParams.append('site', includeDomains.join(','))+ }++ // Fetch results from SearXNG+ const response = await fetch(url.toString(), {+ method: 'GET',+ headers: {+ Accept: 'application/json'+ }+ })++ if (!response.ok) {+ const errorText = await response.text()+ console.error(`SearXNG API error (${response.status}):`, errorText)+ throw new Error(+ `SearXNG API error: ${response.status} ${response.statusText} - ${errorText}`+ )+ }++ const data: SearXNGResponse = await response.json()++ // Separate general results and image results, and limit to maxResults+ const generalResults = data.results+ .filter(result => !result.img_src)+ .slice(0, maxResults)+ const imageResults = data.results+ .filter(result => result.img_src)+ .slice(0, maxResults)++ // Format the results to match the expected SearchResults structure+ return {+ results: generalResults.map(+ (result: SearXNGResult): SearchResultItem => ({+ title: result.title,+ url: result.url,+ content: result.content+ })+ ),+ query: data.query,+ images: imageResults+ .map(result => {+ const imgSrc = result.img_src || ''+ return imgSrc.startsWith('http') ? imgSrc : `${apiUrl}${imgSrc}`+ })+ .filter(Boolean),+ number_of_results: data.number_of_results+ }+ } catch (error) {+ console.error('SearXNG API error:', error)+ throw error+ }+ }+}\ No newline at end of filediff --git a/lib/tools/search/providers/tavily.ts b/lib/tools/search/providers/tavily.tsnew file mode 100644index 00000000..3344cdc4--- /dev/null+++ b/lib/tools/search/providers/tavily.ts@@ -0,0 +1,67 @@+import { SearchResultImage, SearchResults } from '@/lib/types'+import { sanitizeUrl } from '@/lib/utils'+import { BaseSearchProvider } from './base'++export class TavilySearchProvider extends BaseSearchProvider {+ async search(+ query: string,+ maxResults: number = 10,+ searchDepth: 'basic' | 'advanced' = 'basic',+ includeDomains: string[] = [],+ excludeDomains: string[] = []+ ): Promise<SearchResults> {+ const apiKey = process.env.TAVILY_API_KEY+ this.validateApiKey(apiKey, 'TAVILY')++ // Tavily API requires a minimum of 5 characters in the query+ const filledQuery =+ query.length < 5 ? query + ' '.repeat(5 - query.length) : query++ const includeImageDescriptions = true+ const response = await fetch('https://api.tavily.com/search', {+ method: 'POST',+ headers: {+ 'Content-Type': 'application/json'+ },+ body: JSON.stringify({+ api_key: apiKey,+ query: filledQuery,+ max_results: Math.max(maxResults, 5),+ search_depth: searchDepth,+ include_images: true,+ include_image_descriptions: includeImageDescriptions,+ include_answers: true,+ include_domains: includeDomains,+ exclude_domains: excludeDomains+ })+ })++ if (!response.ok) {+ throw new Error(+ `Tavily API error: ${response.status} ${response.statusText}`+ )+ }++ const data = await response.json()+ const processedImages = includeImageDescriptions+ ? data.images+ .map(({ url, description }: { url: string; description: string }) => ({+ url: sanitizeUrl(url),+ description+ }))+ .filter(+ (+ image: SearchResultImage+ ): image is { url: string; description: string } =>+ typeof image === 'object' &&+ image.description !== undefined &&+ image.description !== ''+ )+ : data.images.map((url: string) => sanitizeUrl(url))++ return {+ ...data,+ images: processedImages+ }+ }+}\ No newline at end of file