Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(lockdown): treat lockdown requests as zero data retention (#3433)

Abimael Martell95c06ddb931e
diff --git a/apps/api/src/__tests__/snips/v2/scrape-lockdown.test.ts b/apps/api/src/__tests__/snips/v2/scrape-lockdown.test.ts
index f9b8672cf..dca7a2ee5 100644
--- a/apps/api/src/__tests__/snips/v2/scrape-lockdown.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape-lockdown.test.ts
@@ -1,5 +1,7 @@
import { describeIf, TEST_PRODUCTION } from "../lib";
import { Identity, idmux, scrapeTimeout, scrape, scrapeRaw } from "./lib";
+import { expectScrapeIsCleanedUp } from "../zdr-helpers";
+import { getJobFromGCS } from "../../../lib/gcs-jobs";
import crypto from "crypto";
describeIf(TEST_PRODUCTION)("V2 Scrape Lockdown Mode", () => {
@@ -70,6 +72,31 @@ describeIf(TEST_PRODUCTION)("V2 Scrape Lockdown Mode", () => {
scrapeTimeout,
);
+ test(
+ "should treat lockdown as ZDR — no URL in DB, no GCS blob",
+ async () => {
+ const id = crypto.randomUUID();
+ const url = "https://firecrawl.dev/?lockdownZdr=" + id;
+
+ const seed = await scrape({ url }, identity);
+ expect(seed).toBeDefined();
+ expect(seed.metadata.cacheState).toBe("miss");
+
+ await new Promise(resolve => setTimeout(resolve, 20000));
+
+ const data = await scrape({ url, lockdown: true }, identity);
+ expect(data).toBeDefined();
+ expect(data.metadata.cacheState).toBe("hit");
+
+ const scrapeId = data.metadata.scrapeId!;
+ await expectScrapeIsCleanedUp(scrapeId);
+
+ const gcsJob = await getJobFromGCS(scrapeId);
+ expect(gcsJob).toBeNull();
+ },
+ scrapeTimeout * 2 + 20000,
+ );
+
test(
"should serve cache and skip audio fetch even when audio format is requested",
async () => {
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts
index 2ff80563e..1aa301e4d 100644
--- a/apps/api/src/controllers/v2/scrape.ts
+++ b/apps/api/src/controllers/v2/scrape.ts
@@ -84,7 +84,9 @@ export async function scrapeController(
}
const zeroDataRetention =
- getScrapeZDR(req.acuc?.flags) === "forced" || (req.body.zeroDataRetention ?? false);
+ getScrapeZDR(req.acuc?.flags) === "forced" ||
+ (req.body.zeroDataRetention ?? false) ||
+ (req.body.lockdown ?? false);
const billing: BillingMetadata = req.body.__agentInterop
? { endpoint: "agent" as const, jobId }
: { endpoint: "scrape" as const, jobId };