Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
remove us-generic (#3065)
rowinsg242f5f0ee6d1
diff --git a/apps/api/src/__tests__/snips/v1/types-validation.test.ts b/apps/api/src/__tests__/snips/v1/types-validation.test.tsindex 521136160..81a47c4f1 100644--- a/apps/api/src/__tests__/snips/v1/types-validation.test.ts+++ b/apps/api/src/__tests__/snips/v1/types-validation.test.ts@@ -772,12 +772,12 @@ describe("V1 Types Validation", () => {const input: ScrapeRequestInput = {url: "https://example.com",location: {- country: "us-generic",+ country: "us-whitelist",},};const result = scrapeRequestSchema.parse(input);- expect(result.location?.country).toBe("us-generic");+ expect(result.location?.country).toBe("us-whitelist");});it("should reject location schema with invalid country code", () => {diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 045c77b53..69430ce3e 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -386,7 +386,7 @@ const actionsSchema = z},);-const SPECIAL_COUNTRIES = ["us-generic", "us-whitelist"];+const SPECIAL_COUNTRIES = ["us-whitelist"];const locationSchema = z.object({@@ -401,7 +401,7 @@ const locationSchema = z"Invalid country code. Use a valid ISO 3166-1 alpha-2 country code.",).transform(val => {- if (!val) return "us-generic";+ if (!val) return "us";return val.toLowerCase();}),languages: z.array(z.string()).optional(),diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 2cc7a3617..1911e2a93 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -472,7 +472,7 @@ function transformIframeSelector(selector: string): string {});}-const SPECIAL_COUNTRIES = ["us-generic", "us-whitelist"];+const SPECIAL_COUNTRIES = ["us-whitelist"];const locationSchema = z.object({@@ -487,7 +487,7 @@ const locationSchema = z"Invalid country code. Use a valid ISO 3166-1 alpha-2 country code.",).transform(val => {- if (!val) return "us-generic";+ if (!val) return "us";return val.toLowerCase();}),languages: z.array(z.string()).optional(),diff --git a/apps/api/src/scraper/crawler/sitemap.ts b/apps/api/src/scraper/crawler/sitemap.tsindex dfa68556e..7cfe350b2 100644--- a/apps/api/src/scraper/crawler/sitemap.ts+++ b/apps/api/src/scraper/crawler/sitemap.ts@@ -50,7 +50,7 @@ async function getSitemapXML(options: SitemapScrapeOptions): Promise<string> {}const isLocationSpecified =- options.location && options.location.country !== "us-generic";+ options.location && options.location.country !== "us";const forceEngine: Engine[] = [...(options.maxAge > 0 && useIndex ? ["index" as const] : []),