Skip to content

An AI-powered search engine with a generative UI

fix(search): handle paths in Tavily site-only queries (#1034)

Yoshiki Miuraa7ed58fa2086
diff --git a/lib/tools/search/providers/__tests__/tavily-domains.test.ts b/lib/tools/search/providers/__tests__/tavily-domains.test.ts
index 5ba9dc54..6a8069bc 100644
--- a/lib/tools/search/providers/__tests__/tavily-domains.test.ts
+++ b/lib/tools/search/providers/__tests__/tavily-domains.test.ts
@@ -132,8 +132,74 @@ describe('TavilySearchProvider domains', () => {
expect(body.include_domains).toEqual([])
})
- it('leaves a site operand carrying a path unchanged', async () => {
- const query = 'site:example.com/docs'
+ it('converts a site-only path into a path term and domain entry', async () => {
+ const body = await searchAndReadBody([], [], 'site:example.com/docs')
+
+ expect(body.query).toBe('docs ')
+ expect(body.include_domains).toEqual(['example.com'])
+ })
+
+ it('converts a deep site-only path into terms without its extension', async () => {
+ const body = await searchAndReadBody(
+ [],
+ [],
+ 'site:example.com/a/b/teoria_comercio.pdf'
+ )
+
+ expect(body.query).toBe('a b teoria comercio')
+ expect(body.include_domains).toEqual(['example.com'])
+ })
+
+ it('keeps a dot-prefixed path component and a version slug', async () => {
+ const wellKnown = await searchAndReadBody(
+ [],
+ [],
+ 'site:example.com/.well-known'
+ )
+
+ expect(wellKnown.query).toBe('well known')
+ expect(wellKnown.include_domains).toEqual(['example.com'])
+
+ const versioned = await searchAndReadBody(
+ [],
+ [],
+ 'site:example.com/docs/v1.2'
+ )
+
+ expect(versioned.query).toBe('docs v1 2')
+ expect(versioned.include_domains).toEqual(['example.com'])
+ })
+
+ it('drops an extension carrying digits', async () => {
+ const video = await searchAndReadBody(
+ [],
+ [],
+ 'site:example.com/media/interview.mp4'
+ )
+
+ expect(video.query).toBe('media interview')
+ expect(video.include_domains).toEqual(['example.com'])
+
+ const archive = await searchAndReadBody(
+ [],
+ [],
+ 'site:example.com/archive.7z'
+ )
+
+ expect(archive.query).toBe('archive')
+ expect(archive.include_domains).toEqual(['example.com'])
+ })
+
+ it('leaves a site-only path with an invalid host unchanged', async () => {
+ const query = 'site:edu/docs'
+ const body = await searchAndReadBody([], [], query)
+
+ expect(body.query).toBe(query)
+ expect(body.include_domains).toEqual([])
+ })
+
+ it('leaves a site path combined with search terms unchanged', async () => {
+ const query = 'site:example.com/docs remodeling services'
const body = await searchAndReadBody([], [], query)
expect(body.query).toBe(query)
diff --git a/lib/tools/search/providers/tavily.ts b/lib/tools/search/providers/tavily.ts
index 4414f9e2..77a36327 100644
--- a/lib/tools/search/providers/tavily.ts
+++ b/lib/tools/search/providers/tavily.ts
@@ -12,18 +12,30 @@ const CLOUD_EXCLUDED_DOMAINS = ['instagram.com']
// resolved to an ASCII hostname so internationalized domains survive.
const VALID_DOMAIN_PATTERN = /^(\*\.)?[a-z0-9-]+(\.[a-z0-9-]+)+$/
-// Tavily rejects a query made only of `site:` operators. Operands with a path
-// or port are left alone since include_domains cannot express them.
-const SITE_OPERATOR_PATTERN = /^site:[^\s/:?#@\\]+$/i
+// Tavily rejects a query made only of `site:` operators. Path terms are kept
+// in the query since include_domains can only express their hostnames.
+const SITE_OPERATOR_PATTERN = /^site:[^\s/:?#@\\]+(?:\/\S*)?$/i
-const extractSiteOnlyDomains = (query: string): string[] | null => {
+const extractSiteOnlyOperands = (
+ query: string
+): Array<{ host: string; path: string }> | null => {
const tokens = query.trim().split(/\s+/)
if (tokens.some(token => !SITE_OPERATOR_PATTERN.test(token))) {
return null
}
- return tokens.map(token => token.slice(5))
+ return tokens.map(token => {
+ const operand = token.slice(5)
+ const pathStart = operand.indexOf('/')
+
+ return pathStart === -1
+ ? { host: operand, path: '' }
+ : {
+ host: operand.slice(0, pathStart),
+ path: operand.slice(pathStart + 1)
+ }
+ })
}
const toAsciiHostname = (domain: string): string => {
@@ -44,6 +56,16 @@ const normalizeDomains = (domains: string[]) =>
)
})
+// A trailing extension is dropped, but only when a suffix carrying at least one
+// letter follows a nonempty stem, so `/.well-known` and purely numeric version
+// slugs like `/v1.2` survive.
+const pathToSearchTerms = (path: string): string =>
+ path
+ .replace(/([^/.])\.(?=[a-z0-9]{0,7}[a-z])[a-z0-9]{1,8}$/i, '$1')
+ .replace(/(?:%20|[/_.+-])+/gi, ' ')
+ .replace(/\s+/g, ' ')
+ .trim()
+
export class TavilySearchProvider extends BaseSearchProvider {
async search(
query: string,
@@ -55,12 +77,18 @@ export class TavilySearchProvider extends BaseSearchProvider {
const apiKey = process.env.TAVILY_API_KEY
this.validateApiKey(apiKey, 'TAVILY')
- const siteOnlyDomains = extractSiteOnlyDomains(query)
- const validSiteDomains = siteOnlyDomains
- ? normalizeDomains(siteOnlyDomains)
+ const siteOnlyOperands = extractSiteOnlyOperands(query)
+ const validSiteOperands = siteOnlyOperands
+ ? siteOnlyOperands.flatMap(({ host, path }) => {
+ const [domain] = normalizeDomains([host])
+ return domain ? [{ domain, path }] : []
+ })
: []
- const effectiveQuery = validSiteDomains.length
- ? validSiteDomains.join(' ')
+ const validSiteDomains = validSiteOperands.map(({ domain }) => domain)
+ const effectiveQuery = validSiteOperands.length
+ ? validSiteOperands
+ .map(({ domain, path }) => pathToSearchTerms(path) || domain)
+ .join(' ')
: query
// Tavily API requires a minimum of 5 characters in the query