An AI-powered search engine with a generative UI
TypeScript generative-aigenerative-uinextjsollamareactsearxngshadcn-uitailwindcsstavilytypescriptupstashvercel-ai-sdk
fix(search): drop domain filters Tavily cannot accept (#1010)
Yoshiki Miuracf1d03a57203
diff --git a/lib/tools/search/providers/__tests__/tavily-domains.test.ts b/lib/tools/search/providers/__tests__/tavily-domains.test.tsnew file mode 100644index 00000000..364b6597--- /dev/null+++ b/lib/tools/search/providers/__tests__/tavily-domains.test.ts@@ -0,0 +1,73 @@+import { afterEach, describe, expect, it, vi } from 'vitest'++import { TavilySearchProvider } from '../tavily'++describe('TavilySearchProvider domains', () => {+ afterEach(() => {+ vi.unstubAllEnvs()+ vi.unstubAllGlobals()+ })++ const searchAndReadBody = async (+ includeDomains: string[] = [],+ excludeDomains: string[] = []+ ) => {+ vi.stubEnv('TAVILY_API_KEY', 'test-key')+ const fetchMock = vi.fn(+ async (_input: RequestInfo | URL, _init?: RequestInit) =>+ Response.json({ results: [], images: [] })+ )+ vi.stubGlobal('fetch', fetchMock)++ await new TavilySearchProvider().search(+ 'query',+ 5,+ 'basic',+ includeDomains,+ excludeDomains+ )++ const init = fetchMock.mock.calls[0][1]+ return JSON.parse(init?.body as string)+ }++ it('drops bare TLDs from include_domains', async () => {+ const body = await searchAndReadBody(['edu', 'gov'])++ expect(body.include_domains).toEqual([])+ })++ it('keeps only valid entries in a mixed include_domains list', async () => {+ const body = await searchAndReadBody(['edu', 'mit.edu', 'gov'])++ expect(body.include_domains).toEqual(['mit.edu'])+ })++ it('filters exclude_domains including the cloud-injected domain', async () => {+ vi.stubEnv('MORPHIC_CLOUD_DEPLOYMENT', 'true')++ const body = await searchAndReadBody(+ ['example.org'],+ ['com', 'example.net']+ )++ expect(body.exclude_domains).toEqual(['example.net', 'instagram.com'])+ })++ it('converts internationalized domains to their ascii form', async () => {+ const body = await searchAndReadBody(['münchen.de'], ['*.MÜNCHEN.de'])++ expect(body.include_domains).toEqual(['xn--mnchen-3ya.de'])+ expect(body.exclude_domains).toEqual(['*.xn--mnchen-3ya.de'])+ })++ it('passes valid domains through untouched', async () => {+ const body = await searchAndReadBody(+ ['example.com', '*.example.org'],+ ['example.net']+ )++ expect(body.include_domains).toEqual(['example.com', '*.example.org'])+ expect(body.exclude_domains).toEqual(['example.net'])+ })+})diff --git a/lib/tools/search/providers/tavily.ts b/lib/tools/search/providers/tavily.tsindex 459bbac3..26f04fcc 100644--- a/lib/tools/search/providers/tavily.ts+++ b/lib/tools/search/providers/tavily.ts@@ -7,6 +7,28 @@ import { BaseSearchProvider } from './base'// started surfacing low-value aggregator/social pages (notably Instagram)// that rarely help answer informational queries.const CLOUD_EXCLUDED_DOMAINS = ['instagram.com']+// Tavily rejects the whole request when every entry of a domain list lacks a+// valid suffix, so bare labels are dropped before the call. Entries are first+// resolved to an ASCII hostname so internationalized domains survive.+const VALID_DOMAIN_PATTERN = /^(\*\.)?[a-z0-9-]+(\.[a-z0-9-]+)+$/++const toAsciiHostname = (domain: string): string => {+ try {+ return new URL(`https://${domain.trim()}`).hostname+ } catch {+ return ''+ }+}++const normalizeDomains = (domains: string[]) =>+ domains.map(toAsciiHostname).filter(domain => {+ const suffix = domain.split('.').at(-1) ?? ''+ return (+ VALID_DOMAIN_PATTERN.test(domain) &&+ suffix.length >= 2 &&+ !/^\d+$/.test(suffix)+ )+ })export class TavilySearchProvider extends BaseSearchProvider {async search(@@ -23,10 +45,13 @@ export class TavilySearchProvider extends BaseSearchProvider {const filledQuery =query.length < 5 ? query + ' '.repeat(5 - query.length) : query+ const validIncludeDomains = normalizeDomains(includeDomains)+const isCloudDeployment = process.env.MORPHIC_CLOUD_DEPLOYMENT === 'true'const effectiveExcludeDomains = isCloudDeployment? Array.from(new Set([...excludeDomains, ...CLOUD_EXCLUDED_DOMAINS])): excludeDomains+ const validExcludeDomains = normalizeDomains(effectiveExcludeDomains)const includeImageDescriptions = trueconst response = await fetch('https://api.tavily.com/search', {@@ -42,8 +67,8 @@ export class TavilySearchProvider extends BaseSearchProvider {include_images: true,include_image_descriptions: includeImageDescriptions,include_answers: true,- include_domains: includeDomains,- exclude_domains: effectiveExcludeDomains+ include_domains: validIncludeDomains,+ exclude_domains: validExcludeDomains})})