Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(lockdown): treat lockdown requests as zero data retention (#3433)
Abimael Martell95c06ddb931e
diff --git a/apps/api/src/__tests__/snips/v2/scrape-lockdown.test.ts b/apps/api/src/__tests__/snips/v2/scrape-lockdown.test.tsindex f9b8672cf..dca7a2ee5 100644--- a/apps/api/src/__tests__/snips/v2/scrape-lockdown.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape-lockdown.test.ts@@ -1,5 +1,7 @@import { describeIf, TEST_PRODUCTION } from "../lib";import { Identity, idmux, scrapeTimeout, scrape, scrapeRaw } from "./lib";+import { expectScrapeIsCleanedUp } from "../zdr-helpers";+import { getJobFromGCS } from "../../../lib/gcs-jobs";import crypto from "crypto";describeIf(TEST_PRODUCTION)("V2 Scrape Lockdown Mode", () => {@@ -70,6 +72,31 @@ describeIf(TEST_PRODUCTION)("V2 Scrape Lockdown Mode", () => {scrapeTimeout,);+ test(+ "should treat lockdown as ZDR — no URL in DB, no GCS blob",+ async () => {+ const id = crypto.randomUUID();+ const url = "https://firecrawl.dev/?lockdownZdr=" + id;++ const seed = await scrape({ url }, identity);+ expect(seed).toBeDefined();+ expect(seed.metadata.cacheState).toBe("miss");++ await new Promise(resolve => setTimeout(resolve, 20000));++ const data = await scrape({ url, lockdown: true }, identity);+ expect(data).toBeDefined();+ expect(data.metadata.cacheState).toBe("hit");++ const scrapeId = data.metadata.scrapeId!;+ await expectScrapeIsCleanedUp(scrapeId);++ const gcsJob = await getJobFromGCS(scrapeId);+ expect(gcsJob).toBeNull();+ },+ scrapeTimeout * 2 + 20000,+ );+test("should serve cache and skip audio fetch even when audio format is requested",async () => {diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex 2ff80563e..1aa301e4d 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -84,7 +84,9 @@ export async function scrapeController(}const zeroDataRetention =- getScrapeZDR(req.acuc?.flags) === "forced" || (req.body.zeroDataRetention ?? false);+ getScrapeZDR(req.acuc?.flags) === "forced" ||+ (req.body.zeroDataRetention ?? false) ||+ (req.body.lockdown ?? false);const billing: BillingMetadata = req.body.__agentInterop? { endpoint: "agent" as const, jobId }: { endpoint: "scrape" as const, jobId };