Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): fill in search country from location and location from country (#5193)
Gergő Móricz97df02b1f0b9
diff --git a/apps/api/src/__tests__/snips/v2/search.test.ts b/apps/api/src/__tests__/snips/v2/search.test.tsindex e62212bd6..98acc6235 100644--- a/apps/api/src/__tests__/snips/v2/search.test.ts+++ b/apps/api/src/__tests__/snips/v2/search.test.ts@@ -338,6 +338,58 @@ describeIf(TEST_PRODUCTION || HAS_SEARCH || HAS_PROXY)("Search tests", () => {60000,);+ concurrentIf(TEST_PRODUCTION)(+ "targets the country named by location when country is omitted",+ async () => {+ const res = await search(+ {+ query: "gobierno",+ location: "Mexico",+ },+ identity,+ );+ const topHosts = (res.web ?? [])+ .slice(0, 3)+ .map(result => new URL(result.url).hostname);+ expect(topHosts.some(host => host.endsWith("gob.mx"))).toBe(true);+ },+ 60000,+ );++ concurrentIf(TEST_PRODUCTION)(+ "targets the country's location when location is omitted",+ async () => {+ const res = await search(+ {+ query: "gobierno",+ country: "mx",+ },+ identity,+ );+ const topHosts = (res.web ?? [])+ .slice(0, 3)+ .map(result => new URL(result.url).hostname);+ expect(topHosts.some(host => host.endsWith("gob.mx"))).toBe(true);+ },+ 60000,+ );++ it.concurrent(+ "rejects non-string location",+ async () => {+ const res = await searchWithFailure(+ {+ query: "firecrawl",+ location: 123,+ } as any,+ identity,+ );++ expect(res.error).toBe("Invalid request body");+ },+ 60000,+ );+// SEARXNG-specific pagination testsconcurrentIf(!!config.SEARXNG_ENDPOINT)("searxng respects limit of 2 results",diff --git a/apps/api/src/controllers/v2/search-schema.test.ts b/apps/api/src/controllers/v2/search-schema.test.tsindex 05274b3b5..d31ae0189 100644--- a/apps/api/src/controllers/v2/search-schema.test.ts+++ b/apps/api/src/controllers/v2/search-schema.test.ts@@ -90,3 +90,51 @@ describe("searchRequestSchema highlights", () => {},);});++describe("searchRequestSchema geo targeting", () => {+ it.each([+ [{ location: "Greece" }, "gr", "Greece"],+ [{ location: "Mexico City, Mexico" }, "mx", "Mexico City, Mexico"],+ [+ { location: "San Francisco,California,United States" },+ "us",+ "San Francisco,California,United States",+ ],+ [{ country: "gr" }, "gr", "Greece"],+ [{ country: "MX" }, "MX", "Mexico"],+ [+ { country: "gr", location: "Mexico City, Mexico" },+ "gr",+ "Mexico City, Mexico",+ ],+ [{}, "us", undefined],+ ])("resolves %j to country %s and location %s", (geo, country, location) => {+ const request = searchRequestSchema.parse({ query: "gaming blog", ...geo });++ expect(request.country).toBe(country);+ expect(request.location).toBe(location);+ expect(request.sources[0]).toMatchObject({+ type: "web",+ country,+ location,+ });+ });++ it.each([+ [{ location: "San Francisco" }, undefined, "San Francisco"],+ [{ location: "New Mexico" }, undefined, "New Mexico"],+ [{ location: "Atlanta, Georgia" }, undefined, "Atlanta, Georgia"],+ [{ country: "Greece" }, "Greece", undefined],+ ])(+ "leaves %j unresolved when it names no known country",+ (geo, country, location) => {+ const request = searchRequestSchema.parse({+ query: "gaming blog",+ ...geo,+ });++ expect(request.country).toBe(country);+ expect(request.location).toBe(location);+ },+ );+});diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 5638825d4..752950416 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -2674,8 +2674,24 @@ export const searchRequestSchema = z}, "the gov category cannot be combined with other categories").refine(x => waitForRefine(x.scrapeOptions), waitForRefineOpts).transform(x => {+ // Providers read the country (gl) and the location separately and default+ // each to the US, so fill in whichever one the caller left out. Georgia is+ // never inferred because it is also a US state.+ const locationCountry = x.location?.split(",").at(-1)?.trim().toLowerCase();const country =- x.country !== undefined ? x.country : x.location ? undefined : "us";+ x.country ??+ Object.entries(countries)+ .find(+ ([code, c]) =>+ code !== "GE" && c.name.toLowerCase() === locationCountry,+ )?.[0]+ .toLowerCase() ??+ (x.location ? undefined : "us");+ const location =+ x.location ??+ Object.entries(countries).find(+ ([code]) => code === x.country?.toUpperCase(),+ )?.[1].name;// Transform string array sources to object formatlet sources = x.sources;@@ -2692,7 +2708,7 @@ export const searchRequestSchema = zfilter: x.filter,lang: x.lang,country,- location: x.location,+ location,};case "images":return {@@ -2705,7 +2721,7 @@ export const searchRequestSchema = ztbs: x.tbs,lang: x.lang,country,- location: x.location,+ location,};default:return { type: s as any };@@ -2754,6 +2770,7 @@ export const searchRequestSchema = zreturn {...x,country,+ location,sources,categories,scrapeOptions: extractTransform(x.scrapeOptions),