Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): default v1 proxy to auto when no country is specified (#4622)
Gergő Móriczbbadeb87ba0d
diff --git a/apps/api/src/__tests__/snips/v1/types-validation.test.ts b/apps/api/src/__tests__/snips/v1/types-validation.test.tsindex 26f52dc24..0e98bf04d 100644--- a/apps/api/src/__tests__/snips/v1/types-validation.test.ts+++ b/apps/api/src/__tests__/snips/v1/types-validation.test.ts@@ -34,7 +34,7 @@ describe("V1 Types Validation", () => {const result = scrapeRequestSchema.parse(input);expect(result.url).toBe("https://example.com");expect(result.origin).toBe("api");- expect(result.timeout).toBe(30000);+ expect(result.timeout).toBe(120000); // bumped because default proxy is "auto"expect(result.formats).toEqual(["markdown"]);});@@ -286,7 +286,7 @@ describe("V1 Types Validation", () => {const result = scrapeRequestSchema.parse(input);expect(result.origin).toBe("api");- expect(result.timeout).toBe(30000);+ expect(result.timeout).toBe(120000); // bumped because default proxy is "auto"expect(result.formats).toEqual(["markdown"]);expect(result.onlyMainContent).toBe(true);expect(result.onlyCleanContent).toBe(false);@@ -296,7 +296,7 @@ describe("V1 Types Validation", () => {expect(result.removeBase64Images).toBe(true);expect(result.fastMode).toBe(false);expect(result.blockAds).toBe(true);- expect(result.proxy).toBe("basic");+ expect(result.proxy).toBe("auto");expect(result.storeInCache).toBe(true);});diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex a53698db0..421c469c1 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -542,7 +542,10 @@ const baseScrapeOptions = z.strictObject({fastMode: z.boolean().prefault(false),useMock: z.string().optional(),blockAds: z.boolean().prefault(true),- proxy: z.enum(["basic", "stealth", "enhanced", "auto"]).prefault("basic"),+ // No prefault here: the default is conditional on location (see extractTransform).+ // Requests without a non-default country default to "auto"; requests with one+ // default to "basic".+ proxy: z.enum(["basic", "stealth", "enhanced", "auto"]).optional(),maxAge: z.int().gte(0)@@ -580,6 +583,23 @@ const extractTransformRequired = <T extends ScrapeOptions>(obj: T): T => {};const extractTransform = (obj: ScrapeOptions) => {+ // Proxy default: "auto" when no non-default country is specified, so+ // requests can upgrade to stealth on proxy failures. When a country is+ // specified, keep the historical "basic" default.+ if (obj.proxy === undefined) {+ // Check both location fields: a non-default country in either one counts+ // as specified, even if the other omitted its country (its schema fills+ // in the "us-generic" default, which must not shadow the other field).+ const hasNonDefaultCountry = [+ obj.location?.country,+ obj.geolocation?.country,+ ].some(+ country =>+ country !== undefined && country.toLowerCase() !== "us-generic",+ );+ obj = { ...obj, proxy: hasNonDefaultCountry ? "basic" : "auto" };+ }+// Handle timeoutif ((includesFormat(obj.formats, "extract") ||