An AI-powered search engine with a generative UI
TypeScript generative-aigenerative-uinextjsollamareactsearxngshadcn-uitailwindcsstavilytypescriptupstashvercel-ai-sdk
fix(search): handle paths in Tavily site-only queries (#1034)
Yoshiki Miuraa7ed58fa2086
diff --git a/lib/tools/search/providers/__tests__/tavily-domains.test.ts b/lib/tools/search/providers/__tests__/tavily-domains.test.tsindex 5ba9dc54..6a8069bc 100644--- a/lib/tools/search/providers/__tests__/tavily-domains.test.ts+++ b/lib/tools/search/providers/__tests__/tavily-domains.test.ts@@ -132,8 +132,74 @@ describe('TavilySearchProvider domains', () => {expect(body.include_domains).toEqual([])})- it('leaves a site operand carrying a path unchanged', async () => {- const query = 'site:example.com/docs'+ it('converts a site-only path into a path term and domain entry', async () => {+ const body = await searchAndReadBody([], [], 'site:example.com/docs')++ expect(body.query).toBe('docs ')+ expect(body.include_domains).toEqual(['example.com'])+ })++ it('converts a deep site-only path into terms without its extension', async () => {+ const body = await searchAndReadBody(+ [],+ [],+ 'site:example.com/a/b/teoria_comercio.pdf'+ )++ expect(body.query).toBe('a b teoria comercio')+ expect(body.include_domains).toEqual(['example.com'])+ })++ it('keeps a dot-prefixed path component and a version slug', async () => {+ const wellKnown = await searchAndReadBody(+ [],+ [],+ 'site:example.com/.well-known'+ )++ expect(wellKnown.query).toBe('well known')+ expect(wellKnown.include_domains).toEqual(['example.com'])++ const versioned = await searchAndReadBody(+ [],+ [],+ 'site:example.com/docs/v1.2'+ )++ expect(versioned.query).toBe('docs v1 2')+ expect(versioned.include_domains).toEqual(['example.com'])+ })++ it('drops an extension carrying digits', async () => {+ const video = await searchAndReadBody(+ [],+ [],+ 'site:example.com/media/interview.mp4'+ )++ expect(video.query).toBe('media interview')+ expect(video.include_domains).toEqual(['example.com'])++ const archive = await searchAndReadBody(+ [],+ [],+ 'site:example.com/archive.7z'+ )++ expect(archive.query).toBe('archive')+ expect(archive.include_domains).toEqual(['example.com'])+ })++ it('leaves a site-only path with an invalid host unchanged', async () => {+ const query = 'site:edu/docs'+ const body = await searchAndReadBody([], [], query)++ expect(body.query).toBe(query)+ expect(body.include_domains).toEqual([])+ })++ it('leaves a site path combined with search terms unchanged', async () => {+ const query = 'site:example.com/docs remodeling services'const body = await searchAndReadBody([], [], query)expect(body.query).toBe(query)diff --git a/lib/tools/search/providers/tavily.ts b/lib/tools/search/providers/tavily.tsindex 4414f9e2..77a36327 100644--- a/lib/tools/search/providers/tavily.ts+++ b/lib/tools/search/providers/tavily.ts@@ -12,18 +12,30 @@ const CLOUD_EXCLUDED_DOMAINS = ['instagram.com']// resolved to an ASCII hostname so internationalized domains survive.const VALID_DOMAIN_PATTERN = /^(\*\.)?[a-z0-9-]+(\.[a-z0-9-]+)+$/-// Tavily rejects a query made only of `site:` operators. Operands with a path-// or port are left alone since include_domains cannot express them.-const SITE_OPERATOR_PATTERN = /^site:[^\s/:?#@\\]+$/i+// Tavily rejects a query made only of `site:` operators. Path terms are kept+// in the query since include_domains can only express their hostnames.+const SITE_OPERATOR_PATTERN = /^site:[^\s/:?#@\\]+(?:\/\S*)?$/i-const extractSiteOnlyDomains = (query: string): string[] | null => {+const extractSiteOnlyOperands = (+ query: string+): Array<{ host: string; path: string }> | null => {const tokens = query.trim().split(/\s+/)if (tokens.some(token => !SITE_OPERATOR_PATTERN.test(token))) {return null}- return tokens.map(token => token.slice(5))+ return tokens.map(token => {+ const operand = token.slice(5)+ const pathStart = operand.indexOf('/')++ return pathStart === -1+ ? { host: operand, path: '' }+ : {+ host: operand.slice(0, pathStart),+ path: operand.slice(pathStart + 1)+ }+ })}const toAsciiHostname = (domain: string): string => {@@ -44,6 +56,16 @@ const normalizeDomains = (domains: string[]) =>)})+// A trailing extension is dropped, but only when a suffix carrying at least one+// letter follows a nonempty stem, so `/.well-known` and purely numeric version+// slugs like `/v1.2` survive.+const pathToSearchTerms = (path: string): string =>+ path+ .replace(/([^/.])\.(?=[a-z0-9]{0,7}[a-z])[a-z0-9]{1,8}$/i, '$1')+ .replace(/(?:%20|[/_.+-])+/gi, ' ')+ .replace(/\s+/g, ' ')+ .trim()+export class TavilySearchProvider extends BaseSearchProvider {async search(query: string,@@ -55,12 +77,18 @@ export class TavilySearchProvider extends BaseSearchProvider {const apiKey = process.env.TAVILY_API_KEYthis.validateApiKey(apiKey, 'TAVILY')- const siteOnlyDomains = extractSiteOnlyDomains(query)- const validSiteDomains = siteOnlyDomains- ? normalizeDomains(siteOnlyDomains)+ const siteOnlyOperands = extractSiteOnlyOperands(query)+ const validSiteOperands = siteOnlyOperands+ ? siteOnlyOperands.flatMap(({ host, path }) => {+ const [domain] = normalizeDomains([host])+ return domain ? [{ domain, path }] : []+ }): []- const effectiveQuery = validSiteDomains.length- ? validSiteDomains.join(' ')+ const validSiteDomains = validSiteOperands.map(({ domain }) => domain)+ const effectiveQuery = validSiteOperands.length+ ? validSiteOperands+ .map(({ domain, path }) => pathToSearchTerms(path) || domain)+ .join(' '): query// Tavily API requires a minimum of 5 characters in the query