Skip to content

An AI-powered search engine with a generative UI

fix(search): drop domain filters Tavily cannot accept (#1010)

Yoshiki Miuracf1d03a57203
diff --git a/lib/tools/search/providers/__tests__/tavily-domains.test.ts b/lib/tools/search/providers/__tests__/tavily-domains.test.ts
new file mode 100644
index 00000000..364b6597
--- /dev/null
+++ b/lib/tools/search/providers/__tests__/tavily-domains.test.ts
@@ -0,0 +1,73 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+
+import { TavilySearchProvider } from '../tavily'
+
+describe('TavilySearchProvider domains', () => {
+ afterEach(() => {
+ vi.unstubAllEnvs()
+ vi.unstubAllGlobals()
+ })
+
+ const searchAndReadBody = async (
+ includeDomains: string[] = [],
+ excludeDomains: string[] = []
+ ) => {
+ vi.stubEnv('TAVILY_API_KEY', 'test-key')
+ const fetchMock = vi.fn(
+ async (_input: RequestInfo | URL, _init?: RequestInit) =>
+ Response.json({ results: [], images: [] })
+ )
+ vi.stubGlobal('fetch', fetchMock)
+
+ await new TavilySearchProvider().search(
+ 'query',
+ 5,
+ 'basic',
+ includeDomains,
+ excludeDomains
+ )
+
+ const init = fetchMock.mock.calls[0][1]
+ return JSON.parse(init?.body as string)
+ }
+
+ it('drops bare TLDs from include_domains', async () => {
+ const body = await searchAndReadBody(['edu', 'gov'])
+
+ expect(body.include_domains).toEqual([])
+ })
+
+ it('keeps only valid entries in a mixed include_domains list', async () => {
+ const body = await searchAndReadBody(['edu', 'mit.edu', 'gov'])
+
+ expect(body.include_domains).toEqual(['mit.edu'])
+ })
+
+ it('filters exclude_domains including the cloud-injected domain', async () => {
+ vi.stubEnv('MORPHIC_CLOUD_DEPLOYMENT', 'true')
+
+ const body = await searchAndReadBody(
+ ['example.org'],
+ ['com', 'example.net']
+ )
+
+ expect(body.exclude_domains).toEqual(['example.net', 'instagram.com'])
+ })
+
+ it('converts internationalized domains to their ascii form', async () => {
+ const body = await searchAndReadBody(['münchen.de'], ['*.MÜNCHEN.de'])
+
+ expect(body.include_domains).toEqual(['xn--mnchen-3ya.de'])
+ expect(body.exclude_domains).toEqual(['*.xn--mnchen-3ya.de'])
+ })
+
+ it('passes valid domains through untouched', async () => {
+ const body = await searchAndReadBody(
+ ['example.com', '*.example.org'],
+ ['example.net']
+ )
+
+ expect(body.include_domains).toEqual(['example.com', '*.example.org'])
+ expect(body.exclude_domains).toEqual(['example.net'])
+ })
+})
diff --git a/lib/tools/search/providers/tavily.ts b/lib/tools/search/providers/tavily.ts
index 459bbac3..26f04fcc 100644
--- a/lib/tools/search/providers/tavily.ts
+++ b/lib/tools/search/providers/tavily.ts
@@ -7,6 +7,28 @@ import { BaseSearchProvider } from './base'
// started surfacing low-value aggregator/social pages (notably Instagram)
// that rarely help answer informational queries.
const CLOUD_EXCLUDED_DOMAINS = ['instagram.com']
+// Tavily rejects the whole request when every entry of a domain list lacks a
+// valid suffix, so bare labels are dropped before the call. Entries are first
+// resolved to an ASCII hostname so internationalized domains survive.
+const VALID_DOMAIN_PATTERN = /^(\*\.)?[a-z0-9-]+(\.[a-z0-9-]+)+$/
+
+const toAsciiHostname = (domain: string): string => {
+ try {
+ return new URL(`https://${domain.trim()}`).hostname
+ } catch {
+ return ''
+ }
+}
+
+const normalizeDomains = (domains: string[]) =>
+ domains.map(toAsciiHostname).filter(domain => {
+ const suffix = domain.split('.').at(-1) ?? ''
+ return (
+ VALID_DOMAIN_PATTERN.test(domain) &&
+ suffix.length >= 2 &&
+ !/^\d+$/.test(suffix)
+ )
+ })
export class TavilySearchProvider extends BaseSearchProvider {
async search(
@@ -23,10 +45,13 @@ export class TavilySearchProvider extends BaseSearchProvider {
const filledQuery =
query.length < 5 ? query + ' '.repeat(5 - query.length) : query
+ const validIncludeDomains = normalizeDomains(includeDomains)
+
const isCloudDeployment = process.env.MORPHIC_CLOUD_DEPLOYMENT === 'true'
const effectiveExcludeDomains = isCloudDeployment
? Array.from(new Set([...excludeDomains, ...CLOUD_EXCLUDED_DOMAINS]))
: excludeDomains
+ const validExcludeDomains = normalizeDomains(effectiveExcludeDomains)
const includeImageDescriptions = true
const response = await fetch('https://api.tavily.com/search', {
@@ -42,8 +67,8 @@ export class TavilySearchProvider extends BaseSearchProvider {
include_images: true,
include_image_descriptions: includeImageDescriptions,
include_answers: true,
- include_domains: includeDomains,
- exclude_domains: effectiveExcludeDomains
+ include_domains: validIncludeDomains,
+ exclude_domains: validExcludeDomains
})
})