Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): allow screenshots and pdf actions under zero data retention (#4167)
Rafael Miller7bd7f66de8a5
diff --git a/apps/api/src/__tests__/snips/v2/zdr.test.ts b/apps/api/src/__tests__/snips/v2/zdr.test.tsindex 49e32f3a7..e9c5a66a0 100644--- a/apps/api/src/__tests__/snips/v2/zdr.test.ts+++ b/apps/api/src/__tests__/snips/v2/zdr.test.ts@@ -8,7 +8,7 @@ import {idmux,searchRaw,} from "./lib";-import { describeIf, TEST_PRODUCTION } from "../lib";+import { describeIf, TEST_PRODUCTION, scrapeTimeout } from "../lib";import {getLogs,expectScrapeIsCleanedUp,@@ -150,6 +150,63 @@ describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {);});+ it(+ "should allow screenshots and clean them up",+ async () => {+ const identity = await idmux({+ name: "zdr/screenshot",+ credits: 10000,+ flags: { scrapeZDR: "allowed" },+ });++ const scrape1 = await scrape(+ {+ url: "https://firecrawl.dev",+ formats: ["screenshot"],+ zeroDataRetention: true,+ },+ identity,+ );++ expect(scrape1.screenshot).toBeDefined();++ const gcsJob = await getJobFromGCS(scrape1.metadata.scrapeId!);+ expect(gcsJob).toBeNull();++ await expectScrapeIsCleanedUp(scrape1.metadata.scrapeId!);+ },+ scrapeTimeout,+ );++ it(+ "should allow pdf actions and clean them up",+ async () => {+ const identity = await idmux({+ name: "zdr/pdf-action",+ credits: 10000,+ flags: { scrapeZDR: "allowed" },+ });++ const scrape1 = await scrape(+ {+ url: "https://firecrawl.dev",+ actions: [{ type: "pdf" }],+ zeroDataRetention: true,+ },+ identity,+ );++ expect(scrape1.actions?.pdfs).toBeDefined();+ expect(scrape1.actions?.pdfs?.length).toBe(1);++ const gcsJob = await getJobFromGCS(scrape1.metadata.scrapeId!);+ expect(gcsJob).toBeNull();++ await expectScrapeIsCleanedUp(scrape1.metadata.scrapeId!);+ },+ scrapeTimeout,+ );+it("should allow search when searchZDR is forced", async () => {const identity = await idmux({name: "zdr/search-forced",diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 70be2d8de..fa13ae67d 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -43,7 +43,6 @@ import {NoCachedDataError,LockdownMissError,DNSResolutionError,- ZDRViolationError,PDFPrefetchFailed,DocumentPrefetchFailed,FEPageLoadFailed,@@ -688,30 +687,6 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {"engine.features": Array.from(meta.featureFlags).join(","),});- if (meta.internalOptions.zeroDataRetention) {- if (meta.featureFlags.has("screenshot")) {- throw new ZDRViolationError("screenshot");- }-- if (meta.featureFlags.has("screenshot@fullScreen")) {- throw new ZDRViolationError("screenshot@fullScreen");- }-- if (- meta.options.actions &&- meta.options.actions.find(x => x.type === "screenshot")- ) {- throw new ZDRViolationError("screenshot action");- }-- if (- meta.options.actions &&- meta.options.actions.find(x => x.type === "pdf")- ) {- throw new ZDRViolationError("pdf action");- }- }-// TODO: handle sitemap data, see WebScraper/index.ts:280// TODO: ScrapeEvents