Skip to content

An AI-powered search engine with a generative UI

fix(search): rewrite site-only Tavily queries into a domain filter (#1020)

Yoshiki Miura34a4279271cd
diff --git a/lib/tools/search/providers/__tests__/tavily-domains.test.ts b/lib/tools/search/providers/__tests__/tavily-domains.test.ts
index 364b6597..5ba9dc54 100644
--- a/lib/tools/search/providers/__tests__/tavily-domains.test.ts
+++ b/lib/tools/search/providers/__tests__/tavily-domains.test.ts
@@ -10,7 +10,8 @@ describe('TavilySearchProvider domains', () => {
const searchAndReadBody = async (
includeDomains: string[] = [],
- excludeDomains: string[] = []
+ excludeDomains: string[] = [],
+ query: string = 'query'
) => {
vi.stubEnv('TAVILY_API_KEY', 'test-key')
const fetchMock = vi.fn(
@@ -20,7 +21,7 @@ describe('TavilySearchProvider domains', () => {
vi.stubGlobal('fetch', fetchMock)
await new TavilySearchProvider().search(
- 'query',
+ query,
5,
'basic',
includeDomains,
@@ -70,4 +71,79 @@ describe('TavilySearchProvider domains', () => {
expect(body.include_domains).toEqual(['example.com', '*.example.org'])
expect(body.exclude_domains).toEqual(['example.net'])
})
+
+ it('converts a site-only query into a domain term and include_domains entry', async () => {
+ const body = await searchAndReadBody([], [], 'site:example.com')
+
+ expect(body.query).toBe('example.com')
+ expect(body.include_domains).toEqual(['example.com'])
+ })
+
+ it('converts multiple site-only operators into domain terms', async () => {
+ const body = await searchAndReadBody(
+ ['existing.net'],
+ [],
+ 'site:example.com SITE:example.org'
+ )
+
+ expect(body.query).toBe('example.com example.org')
+ expect(body.include_domains).toEqual([
+ 'existing.net',
+ 'example.com',
+ 'example.org'
+ ])
+ })
+
+ it('leaves a site operator combined with search terms unchanged', async () => {
+ const query = 'site:example.com remodeling services'
+ const body = await searchAndReadBody([], [], query)
+
+ expect(body.query).toBe(query)
+ expect(body.include_domains).toEqual([])
+ })
+
+ it('leaves a query without site operators unchanged', async () => {
+ const query = 'remodeling services'
+ const body = await searchAndReadBody([], [], query)
+
+ expect(body.query).toBe(query)
+ expect(body.include_domains).toEqual([])
+ })
+
+ it('leaves a site-only query whose domains are all invalid unchanged', async () => {
+ const body = await searchAndReadBody([], [], 'site:edu')
+
+ expect(body.query).toBe('site:edu')
+ expect(body.include_domains).toEqual([])
+ })
+
+ it('punycodes an internationalized site-only operand', async () => {
+ const body = await searchAndReadBody([], [], 'site:münchen.de')
+
+ expect(body.query).toBe('xn--mnchen-3ya.de')
+ expect(body.include_domains).toEqual(['xn--mnchen-3ya.de'])
+ })
+
+ it('leaves a site operand carrying a port unchanged', async () => {
+ const query = 'site:example.com:8080'
+ const body = await searchAndReadBody([], [], query)
+
+ expect(body.query).toBe(query)
+ expect(body.include_domains).toEqual([])
+ })
+
+ it('leaves a site operand carrying a path unchanged', async () => {
+ const query = 'site:example.com/docs'
+ const body = await searchAndReadBody([], [], query)
+
+ expect(body.query).toBe(query)
+ expect(body.include_domains).toEqual([])
+ })
+
+ it('keeps only the valid domains of a mixed site-only query', async () => {
+ const body = await searchAndReadBody([], [], 'site:example.com site:edu')
+
+ expect(body.query).toBe('example.com')
+ expect(body.include_domains).toEqual(['example.com'])
+ })
})
diff --git a/lib/tools/search/providers/tavily.ts b/lib/tools/search/providers/tavily.ts
index 26f04fcc..4414f9e2 100644
--- a/lib/tools/search/providers/tavily.ts
+++ b/lib/tools/search/providers/tavily.ts
@@ -12,6 +12,20 @@ const CLOUD_EXCLUDED_DOMAINS = ['instagram.com']
// resolved to an ASCII hostname so internationalized domains survive.
const VALID_DOMAIN_PATTERN = /^(\*\.)?[a-z0-9-]+(\.[a-z0-9-]+)+$/
+// Tavily rejects a query made only of `site:` operators. Operands with a path
+// or port are left alone since include_domains cannot express them.
+const SITE_OPERATOR_PATTERN = /^site:[^\s/:?#@\\]+$/i
+
+const extractSiteOnlyDomains = (query: string): string[] | null => {
+ const tokens = query.trim().split(/\s+/)
+
+ if (tokens.some(token => !SITE_OPERATOR_PATTERN.test(token))) {
+ return null
+ }
+
+ return tokens.map(token => token.slice(5))
+}
+
const toAsciiHostname = (domain: string): string => {
try {
return new URL(`https://${domain.trim()}`).hostname
@@ -41,11 +55,24 @@ export class TavilySearchProvider extends BaseSearchProvider {
const apiKey = process.env.TAVILY_API_KEY
this.validateApiKey(apiKey, 'TAVILY')
+ const siteOnlyDomains = extractSiteOnlyDomains(query)
+ const validSiteDomains = siteOnlyDomains
+ ? normalizeDomains(siteOnlyDomains)
+ : []
+ const effectiveQuery = validSiteDomains.length
+ ? validSiteDomains.join(' ')
+ : query
+
// Tavily API requires a minimum of 5 characters in the query
const filledQuery =
- query.length < 5 ? query + ' '.repeat(5 - query.length) : query
+ effectiveQuery.length < 5
+ ? effectiveQuery + ' '.repeat(5 - effectiveQuery.length)
+ : effectiveQuery
- const validIncludeDomains = normalizeDomains(includeDomains)
+ const validIncludeDomains = [
+ ...normalizeDomains(includeDomains),
+ ...validSiteDomains
+ ]
const isCloudDeployment = process.env.MORPHIC_CLOUD_DEPLOYMENT === 'true'
const effectiveExcludeDomains = isCloudDeployment