Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(lockdown): Update maxAge for Lockdown Queries (#3411)
Abimael Martellcc6a286deedf
diff --git a/apps/api/src/__tests__/snips/v2/types-validation.test.ts b/apps/api/src/__tests__/snips/v2/types-validation.test.tsindex a75d19e3a..98c91032d 100644--- a/apps/api/src/__tests__/snips/v2/types-validation.test.ts+++ b/apps/api/src/__tests__/snips/v2/types-validation.test.ts@@ -470,14 +470,14 @@ describe("V2 Types Validation", () => {expect(result.lockdown).toBe(true);});- it("should default maxAge to MAX_SAFE_INTEGER when lockdown is true and maxAge is unset", () => {+ it("should default maxAge to ~2 years when lockdown is true and maxAge is unset", () => {const input: ScrapeRequestInput = {url: "https://example.com",lockdown: true,};const result = scrapeRequestSchema.parse(input);- expect(result.maxAge).toBe(Number.MAX_SAFE_INTEGER);+ expect(result.maxAge).toBe(2 * 365 * 24 * 60 * 60 * 1000);});it("should preserve maxAge when lockdown is true and maxAge is provided", () => {diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 1c869a738..961c123f7 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -644,7 +644,10 @@ const extractTransformImpl = <T extends ScrapeOptionsBase | undefined>(}if (obj.lockdown && obj.maxAge === undefined) {- result = { ...result, maxAge: Number.MAX_SAFE_INTEGER };+ // 2 years in ms. Number.MAX_SAFE_INTEGER lands ~285,000 years which+ // overflows Postgres TIMESTAMP arithmetic in the index lookup and silently+ // returns no rows. 2 years covers any practical cache retention window.+ result = { ...result, maxAge: 2 * 365 * 24 * 60 * 60 * 1000 };}return result as T extends undefined ? undefined : T;