An AI-powered search engine with a generative UI
TypeScript generative-aigenerative-uinextjsollamareactsearxngshadcn-uitailwindcsstavilytypescriptupstashvercel-ai-sdk
fix(search): rewrite site-only Tavily queries into a domain filter (#1020)
Yoshiki Miura34a4279271cd
diff --git a/lib/tools/search/providers/__tests__/tavily-domains.test.ts b/lib/tools/search/providers/__tests__/tavily-domains.test.tsindex 364b6597..5ba9dc54 100644--- a/lib/tools/search/providers/__tests__/tavily-domains.test.ts+++ b/lib/tools/search/providers/__tests__/tavily-domains.test.ts@@ -10,7 +10,8 @@ describe('TavilySearchProvider domains', () => {const searchAndReadBody = async (includeDomains: string[] = [],- excludeDomains: string[] = []+ excludeDomains: string[] = [],+ query: string = 'query') => {vi.stubEnv('TAVILY_API_KEY', 'test-key')const fetchMock = vi.fn(@@ -20,7 +21,7 @@ describe('TavilySearchProvider domains', () => {vi.stubGlobal('fetch', fetchMock)await new TavilySearchProvider().search(- 'query',+ query,5,'basic',includeDomains,@@ -70,4 +71,79 @@ describe('TavilySearchProvider domains', () => {expect(body.include_domains).toEqual(['example.com', '*.example.org'])expect(body.exclude_domains).toEqual(['example.net'])})++ it('converts a site-only query into a domain term and include_domains entry', async () => {+ const body = await searchAndReadBody([], [], 'site:example.com')++ expect(body.query).toBe('example.com')+ expect(body.include_domains).toEqual(['example.com'])+ })++ it('converts multiple site-only operators into domain terms', async () => {+ const body = await searchAndReadBody(+ ['existing.net'],+ [],+ 'site:example.com SITE:example.org'+ )++ expect(body.query).toBe('example.com example.org')+ expect(body.include_domains).toEqual([+ 'existing.net',+ 'example.com',+ 'example.org'+ ])+ })++ it('leaves a site operator combined with search terms unchanged', async () => {+ const query = 'site:example.com remodeling services'+ const body = await searchAndReadBody([], [], query)++ expect(body.query).toBe(query)+ expect(body.include_domains).toEqual([])+ })++ it('leaves a query without site operators unchanged', async () => {+ const query = 'remodeling services'+ const body = await searchAndReadBody([], [], query)++ expect(body.query).toBe(query)+ expect(body.include_domains).toEqual([])+ })++ it('leaves a site-only query whose domains are all invalid unchanged', async () => {+ const body = await searchAndReadBody([], [], 'site:edu')++ expect(body.query).toBe('site:edu')+ expect(body.include_domains).toEqual([])+ })++ it('punycodes an internationalized site-only operand', async () => {+ const body = await searchAndReadBody([], [], 'site:münchen.de')++ expect(body.query).toBe('xn--mnchen-3ya.de')+ expect(body.include_domains).toEqual(['xn--mnchen-3ya.de'])+ })++ it('leaves a site operand carrying a port unchanged', async () => {+ const query = 'site:example.com:8080'+ const body = await searchAndReadBody([], [], query)++ expect(body.query).toBe(query)+ expect(body.include_domains).toEqual([])+ })++ it('leaves a site operand carrying a path unchanged', async () => {+ const query = 'site:example.com/docs'+ const body = await searchAndReadBody([], [], query)++ expect(body.query).toBe(query)+ expect(body.include_domains).toEqual([])+ })++ it('keeps only the valid domains of a mixed site-only query', async () => {+ const body = await searchAndReadBody([], [], 'site:example.com site:edu')++ expect(body.query).toBe('example.com')+ expect(body.include_domains).toEqual(['example.com'])+ })})diff --git a/lib/tools/search/providers/tavily.ts b/lib/tools/search/providers/tavily.tsindex 26f04fcc..4414f9e2 100644--- a/lib/tools/search/providers/tavily.ts+++ b/lib/tools/search/providers/tavily.ts@@ -12,6 +12,20 @@ const CLOUD_EXCLUDED_DOMAINS = ['instagram.com']// resolved to an ASCII hostname so internationalized domains survive.const VALID_DOMAIN_PATTERN = /^(\*\.)?[a-z0-9-]+(\.[a-z0-9-]+)+$/+// Tavily rejects a query made only of `site:` operators. Operands with a path+// or port are left alone since include_domains cannot express them.+const SITE_OPERATOR_PATTERN = /^site:[^\s/:?#@\\]+$/i++const extractSiteOnlyDomains = (query: string): string[] | null => {+ const tokens = query.trim().split(/\s+/)++ if (tokens.some(token => !SITE_OPERATOR_PATTERN.test(token))) {+ return null+ }++ return tokens.map(token => token.slice(5))+}+const toAsciiHostname = (domain: string): string => {try {return new URL(`https://${domain.trim()}`).hostname@@ -41,11 +55,24 @@ export class TavilySearchProvider extends BaseSearchProvider {const apiKey = process.env.TAVILY_API_KEYthis.validateApiKey(apiKey, 'TAVILY')+ const siteOnlyDomains = extractSiteOnlyDomains(query)+ const validSiteDomains = siteOnlyDomains+ ? normalizeDomains(siteOnlyDomains)+ : []+ const effectiveQuery = validSiteDomains.length+ ? validSiteDomains.join(' ')+ : query+// Tavily API requires a minimum of 5 characters in the queryconst filledQuery =- query.length < 5 ? query + ' '.repeat(5 - query.length) : query+ effectiveQuery.length < 5+ ? effectiveQuery + ' '.repeat(5 - effectiveQuery.length)+ : effectiveQuery- const validIncludeDomains = normalizeDomains(includeDomains)+ const validIncludeDomains = [+ ...normalizeDomains(includeDomains),+ ...validSiteDomains+ ]const isCloudDeployment = process.env.MORPHIC_CLOUD_DEPLOYMENT === 'true'const effectiveExcludeDomains = isCloudDeployment