Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): allow screenshots and pdf actions under zero data retention (#4167)

Rafael Miller7bd7f66de8a5
diff --git a/apps/api/src/__tests__/snips/v2/zdr.test.ts b/apps/api/src/__tests__/snips/v2/zdr.test.ts
index 49e32f3a7..e9c5a66a0 100644
--- a/apps/api/src/__tests__/snips/v2/zdr.test.ts
+++ b/apps/api/src/__tests__/snips/v2/zdr.test.ts
@@ -8,7 +8,7 @@ import {
idmux,
searchRaw,
} from "./lib";
-import { describeIf, TEST_PRODUCTION } from "../lib";
+import { describeIf, TEST_PRODUCTION, scrapeTimeout } from "../lib";
import {
getLogs,
expectScrapeIsCleanedUp,
@@ -150,6 +150,63 @@ describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {
);
});
+ it(
+ "should allow screenshots and clean them up",
+ async () => {
+ const identity = await idmux({
+ name: "zdr/screenshot",
+ credits: 10000,
+ flags: { scrapeZDR: "allowed" },
+ });
+
+ const scrape1 = await scrape(
+ {
+ url: "https://firecrawl.dev",
+ formats: ["screenshot"],
+ zeroDataRetention: true,
+ },
+ identity,
+ );
+
+ expect(scrape1.screenshot).toBeDefined();
+
+ const gcsJob = await getJobFromGCS(scrape1.metadata.scrapeId!);
+ expect(gcsJob).toBeNull();
+
+ await expectScrapeIsCleanedUp(scrape1.metadata.scrapeId!);
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "should allow pdf actions and clean them up",
+ async () => {
+ const identity = await idmux({
+ name: "zdr/pdf-action",
+ credits: 10000,
+ flags: { scrapeZDR: "allowed" },
+ });
+
+ const scrape1 = await scrape(
+ {
+ url: "https://firecrawl.dev",
+ actions: [{ type: "pdf" }],
+ zeroDataRetention: true,
+ },
+ identity,
+ );
+
+ expect(scrape1.actions?.pdfs).toBeDefined();
+ expect(scrape1.actions?.pdfs?.length).toBe(1);
+
+ const gcsJob = await getJobFromGCS(scrape1.metadata.scrapeId!);
+ expect(gcsJob).toBeNull();
+
+ await expectScrapeIsCleanedUp(scrape1.metadata.scrapeId!);
+ },
+ scrapeTimeout,
+ );
+
it("should allow search when searchZDR is forced", async () => {
const identity = await idmux({
name: "zdr/search-forced",
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 70be2d8de..fa13ae67d 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -43,7 +43,6 @@ import {
NoCachedDataError,
LockdownMissError,
DNSResolutionError,
- ZDRViolationError,
PDFPrefetchFailed,
DocumentPrefetchFailed,
FEPageLoadFailed,
@@ -688,30 +687,6 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
"engine.features": Array.from(meta.featureFlags).join(","),
});
- if (meta.internalOptions.zeroDataRetention) {
- if (meta.featureFlags.has("screenshot")) {
- throw new ZDRViolationError("screenshot");
- }
-
- if (meta.featureFlags.has("screenshot@fullScreen")) {
- throw new ZDRViolationError("screenshot@fullScreen");
- }
-
- if (
- meta.options.actions &&
- meta.options.actions.find(x => x.type === "screenshot")
- ) {
- throw new ZDRViolationError("screenshot action");
- }
-
- if (
- meta.options.actions &&
- meta.options.actions.find(x => x.type === "pdf")
- ) {
- throw new ZDRViolationError("pdf action");
- }
- }
-
// TODO: handle sitemap data, see WebScraper/index.ts:280
// TODO: ScrapeEvents