Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): refuse audio, video and product formats under zero data retention (#5423)
Gergő Móriczcccc2a6e554b
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex 193152b2a..f6cef5367 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -333,6 +333,17 @@ jobs:--reporter=defaultworking-directory: apps/api+ - name: Run audio, video and product format zero data retention regression tests+ if: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'+ run: >-+ pnpm exec vitest run+ src/scraper/scrapeURL/transformers/__tests__/audio.test.ts+ src/scraper/scrapeURL/transformers/__tests__/video.test.ts+ src/scraper/scrapeURL/transformers/__tests__/product.test.ts+ src/lib/scrape-billing.test.ts+ --reporter=default+ working-directory: apps/api+- name: Run country check route regression testsif: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'run: pnpm exec vitest run src/__tests__/routes/country-check.routes.test.ts --reporter=defaultdiff --git a/apps/api/src/__tests__/snips/v2/zdr.test.ts b/apps/api/src/__tests__/snips/v2/zdr.test.tsindex ecb18b887..81a2749eb 100644--- a/apps/api/src/__tests__/snips/v2/zdr.test.ts+++ b/apps/api/src/__tests__/snips/v2/zdr.test.ts@@ -7,8 +7,17 @@ import {zdrcleaner,idmux,searchRaw,+ Identity,} from "./lib";-import { describeIf, itIf, TEST_PRODUCTION, scrapeTimeout } from "../lib";+import {+ describeIf,+ itIf,+ concurrentIf,+ TEST_PRODUCTION,+ TEST_SUITE_WEBSITE,+ HAS_PRODUCT_SERVICE,+ scrapeTimeout,+} from "../lib";import {getLogs,expectScrapeIsCleanedUp,@@ -224,4 +233,145 @@ describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {expect(response.statusCode).toBe(200);expect(response.body.success).toBe(true);}, 60000);++ describe.each(scopes)("%s audio, video and product formats", scope => {+ let identity: Identity;++ beforeAll(async () => {+ identity = await idmux({+ name: `zdr/${scope}/formats`,+ credits: 10000,+ flags: {+ scrapeZDR: scope === "Team-scoped" ? "forced" : "allowed",+ },+ });+ }, scrapeTimeout);++ const zeroDataRetention = scope === "Request-scoped" ? true : undefined;++ it.concurrent(+ "refuses the audio format without attempting the download",+ async () => {+ // Not a supported media URL: a download attempt would fail the scrape.+ const response = await scrape(+ {+ url: "https://firecrawl.dev",+ formats: ["markdown", "audio"],+ zeroDataRetention,+ },+ identity,+ );++ expect(response.markdown).toBeDefined();+ expect(response.audio).toBeUndefined();+ expect(response.warning).toContain(+ "Audio format is not supported with zero data retention.",+ );+ },+ scrapeTimeout,+ );++ it.concurrent(+ "refuses the video format without attempting the download",+ async () => {+ const response = await scrape(+ {+ url: "https://firecrawl.dev",+ formats: ["markdown", "video"],+ zeroDataRetention,+ },+ identity,+ );++ expect(response.markdown).toBeDefined();+ expect(response.video).toBeUndefined();+ expect(response.videos).toBeUndefined();+ expect(response.warning).toContain(+ "Video format is not supported with zero data retention.",+ );+ },+ scrapeTimeout,+ );++ it.concurrent(+ "refuses the product format",+ async () => {+ const response = await scrape(+ {+ url: `${TEST_SUITE_WEBSITE}/product`,+ formats: ["markdown", { type: "product" }],+ zeroDataRetention,+ },+ identity,+ );++ expect(response.markdown).toBeDefined();+ expect(response.product).toBeUndefined();+ expect(response.warning).toContain(+ "Product format is not supported with zero data retention.",+ );+ },+ scrapeTimeout,+ );+ });++ describe("audio, video and product formats without zero data retention on the request", () => {+ let identity: Identity;++ beforeAll(async () => {+ identity = await idmux({+ name: "zdr/allowed/formats-without-zdr",+ credits: 10000,+ flags: { scrapeZDR: "allowed" },+ });+ }, scrapeTimeout);++ it.concurrent(+ "still returns audio",+ async () => {+ const response = await scrape(+ {+ url: "https://www.youtube.com/watch?v=dQw4w9WgXcQ",+ formats: ["audio"],+ },+ identity,+ );++ expect(response.audio).toMatch(/^https:\/\//);+ },+ scrapeTimeout,+ );++ it.concurrent(+ "still returns video",+ async () => {+ const response = await scrape(+ {+ url: "https://www.youtube.com/watch?v=dQw4w9WgXcQ",+ formats: ["video"],+ },+ identity,+ );++ expect(response.video).toMatch(/^https:\/\//);+ },+ scrapeTimeout * 2,+ );++ concurrentIf(HAS_PRODUCT_SERVICE)(+ "still returns the product",+ async () => {+ const response = await scrape(+ {+ url: `${TEST_SUITE_WEBSITE}/product`,+ formats: [{ type: "product" }],+ },+ identity,+ );++ expect(response.product?.title).toBe("Firecrawl Test Widget");+ },+ scrapeTimeout,+ );+ });});diff --git a/apps/api/src/lib/scrape-billing.test.ts b/apps/api/src/lib/scrape-billing.test.tsindex fd25cebc4..46b3f1648 100644--- a/apps/api/src/lib/scrape-billing.test.ts+++ b/apps/api/src/lib/scrape-billing.test.ts@@ -480,6 +480,16 @@ describe("calculateCreditsToBeBilled — zero data retention", () => {it("is not added on top of lockdown", async () => {expect(await bill({ lockdown: true }, true)).toBe(5);});++ it.each(["audio", "video"])(+ "drops the %s surcharge it refuses, but not under lockdown",+ async type => {+ const media = { formats: [{ type }] };+ expect(await bill(media, false)).toBe(5);+ expect(await bill(media, true)).toBe(2);+ expect(await bill({ ...media, lockdown: true }, true)).toBe(9);+ },+ );});describe("calculateThreatScanCredits", () => {diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.tsindex 7df98515e..585c54dee 100644--- a/apps/api/src/lib/scrape-billing.ts+++ b/apps/api/src/lib/scrape-billing.ts@@ -216,11 +216,16 @@ export async function calculateCreditsToBeBilled(creditsToBeBilled += 4;}- if (hasFormatOfType(options.formats, "audio")) {+ // Not billed when zero data retention refuses them (transformers/audio.ts,+ // video.ts). Lockdown skips the fetch before that check and keeps the surcharge.+ const mediaFormatsRefused =+ !!internalOptions.zeroDataRetention && !options.lockdown;++ if (hasFormatOfType(options.formats, "audio") && !mediaFormatsRefused) {creditsToBeBilled += 4;}- if (hasFormatOfType(options.formats, "video")) {+ if (hasFormatOfType(options.formats, "video") && !mediaFormatsRefused) {creditsToBeBilled += 4;}diff --git a/apps/api/src/scraper/scrapeURL/transformers/__tests__/audio.test.ts b/apps/api/src/scraper/scrapeURL/transformers/__tests__/audio.test.tsindex ec6591116..3640f0479 100644--- a/apps/api/src/scraper/scrapeURL/transformers/__tests__/audio.test.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/__tests__/audio.test.ts@@ -53,6 +53,29 @@ describe("fetchAudio lockdown guard", () => {expect(result.audio).toBeUndefined();});+ it("refuses the audio format under zero data retention without calling avgrab", async () => {+ const fetchSpy = mockSuccessfulAvgrab();++ const meta: any = {+ url: "https://example.com/audio",+ options: {+ lockdown: false,+ formats: [{ type: "audio" }],+ },+ internalOptions: { zeroDataRetention: true },+ logger: { warn: vi.fn(), info: vi.fn(), debug: vi.fn() },+ };+ const document: any = {};++ const result = await fetchAudio(meta, document);++ expect(fetchSpy).not.toHaveBeenCalled();+ expect(result.audio).toBeUndefined();+ expect(result.warning).toBe(+ "Audio format is not supported with zero data retention.",+ );+ });+it("returns early when audio format is not requested regardless of lockdown", async () => {const fetchSpy = vi.fn();global.fetch = fetchSpy as any;@@ -92,6 +115,7 @@ describe("fetchAudio lockdown guard", () => {lockdown: false,formats: [{ type: "audio" }],},+ internalOptions: { zeroDataRetention: false },logger: { warn: vi.fn(), info: vi.fn(), debug: vi.fn() },};const document: any = {};@@ -140,6 +164,7 @@ describe("fetchAudio lockdown guard", () => {lockdown: false,formats: [{ type: "audio" }],},+ internalOptions: { zeroDataRetention: false },logger: { warn: vi.fn(), info: vi.fn(), debug: vi.fn() },};@@ -175,6 +200,7 @@ describe("fetchAudio lockdown guard", () => {lockdown: false,formats: [{ type: "audio" }],},+ internalOptions: { zeroDataRetention: false },logger: { warn: vi.fn(), info: vi.fn(), debug: vi.fn() },};@@ -195,6 +221,7 @@ describe("fetchAudio lockdown guard", () => {lockdown: false,formats: [{ type: "audio" }],},+ internalOptions: { zeroDataRetention: false },logger: { warn: vi.fn(), info: vi.fn(), debug: vi.fn() },};const document: any = {};diff --git a/apps/api/src/scraper/scrapeURL/transformers/__tests__/product.test.ts b/apps/api/src/scraper/scrapeURL/transformers/__tests__/product.test.tsindex b501884a2..75cc69cc2 100644--- a/apps/api/src/scraper/scrapeURL/transformers/__tests__/product.test.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/__tests__/product.test.ts@@ -11,11 +11,15 @@ describe("fetchProduct", () => {vi.clearAllMocks();});- function baseMeta(formats: any[] = [{ type: "product" }]) {+ function baseMeta(+ formats: any[] = [{ type: "product" }],+ zeroDataRetention = false,+ ) {return {url: "https://shop.test/p",rewrittenUrl: "https://shop.test/p",options: { formats },+ internalOptions: { zeroDataRetention },logger: { warn: vi.fn(), info: vi.fn(), debug: vi.fn(), error: vi.fn() },} as any;}@@ -36,6 +40,27 @@ describe("fetchProduct", () => {expect(out.warning).toMatch(/not available/i);});+ it("refuses the product format under zero data retention without calling the service", async () => {+ config.PRODUCT_EXTRACTION_SERVICE_URL = "https://product.internal";+ const fetchSpy = vi.fn();+ global.fetch = fetchSpy as any;+ const document: any = {+ rawHtml: "<html>jsonld</html>",+ metadata: { url: "https://shop.test/p" },+ };++ const out = await fetchProduct(+ baseMeta([{ type: "product" }], true),+ document,+ );++ expect(fetchSpy).not.toHaveBeenCalled();+ expect(out.product).toBeUndefined();+ expect(out.warning).toBe(+ "Product format is not supported with zero data retention.",+ );+ });+it("posts rawHtml to the service and sets document.product", async () => {config.PRODUCT_EXTRACTION_SERVICE_URL = "https://product.internal";const product = { title: "X", url: "https://shop.test/p", variants: [] };diff --git a/apps/api/src/scraper/scrapeURL/transformers/__tests__/video.test.ts b/apps/api/src/scraper/scrapeURL/transformers/__tests__/video.test.tsindex a000ee9c8..6ca7a27d7 100644--- a/apps/api/src/scraper/scrapeURL/transformers/__tests__/video.test.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/__tests__/video.test.ts@@ -62,6 +62,7 @@ describe("fetchVideo", () => {lockdown: false,formats: [{ type: "video" }],},+ internalOptions: { zeroDataRetention: false },logger: { warn: vi.fn(), info: vi.fn(), debug: vi.fn() },};const document: any = { rawHtml: "<html></html>" };@@ -104,6 +105,36 @@ describe("fetchVideo", () => {expect(result.video).toBeUndefined();});+ it.each([+ "https://www.youtube.com/watch?v=abc123",+ "https://example.com/product",+ ])(+ "refuses the video format under zero data retention without calling avgrab (%s)",+ async url => {+ const fetchSpy = mockSuccessfulAvgrab();++ const meta: any = {+ url,+ options: {+ lockdown: false,+ formats: [{ type: "video" }],+ },+ internalOptions: { zeroDataRetention: true },+ logger: { warn: vi.fn(), info: vi.fn(), debug: vi.fn() },+ };+ const document: any = { rawHtml: "<html></html>" };++ const result = await fetchVideo(meta, document);++ expect(fetchSpy).not.toHaveBeenCalled();+ expect(result.video).toBeUndefined();+ expect(result.videos).toBeUndefined();+ expect(result.warning).toBe(+ "Video format is not supported with zero data retention.",+ );+ },+ );+it("returns early when video format is not requested regardless of lockdown", async () => {const fetchSpy = vi.fn();global.fetch = fetchSpy as any;@@ -114,6 +145,7 @@ describe("fetchVideo", () => {lockdown: false,formats: [{ type: "markdown" }],},+ internalOptions: { zeroDataRetention: false },logger: { warn: vi.fn(), info: vi.fn(), debug: vi.fn() },};const document: any = { markdown: "cached" };@@ -134,6 +166,7 @@ describe("fetchVideo", () => {lockdown: false,formats: [{ type: "video" }],},+ internalOptions: { zeroDataRetention: false },logger: { warn: vi.fn(), info: vi.fn(), debug: vi.fn() },};const document: any = {};@@ -167,6 +200,7 @@ describe("fetchVideo", () => {lockdown: false,formats: [{ type: "video" }],},+ internalOptions: { zeroDataRetention: false },logger: { warn: vi.fn(), info: vi.fn(), debug: vi.fn() },};const document: any = {};@@ -212,6 +246,7 @@ describe("fetchVideo", () => {lockdown: false,formats: [{ type: "video" }],},+ internalOptions: { zeroDataRetention: false },logger: { warn: vi.fn(), info: vi.fn(), debug: vi.fn() },};const document: any = {};@@ -253,6 +288,7 @@ describe("fetchVideo", () => {lockdown: false,formats: [{ type: "video" }],},+ internalOptions: { zeroDataRetention: false },logger: { warn: vi.fn(), info: vi.fn(), debug: vi.fn() },};const document: any = {};@@ -276,6 +312,7 @@ describe("fetchVideo", () => {lockdown: false,formats: [{ type: "video" }],},+ internalOptions: { zeroDataRetention: false },logger: { warn: vi.fn(), info: vi.fn(), debug: vi.fn() },};const document: any = {};@@ -302,6 +339,7 @@ describe("fetchVideo", () => {lockdown: false,formats: [{ type: "video" }],},+ internalOptions: { zeroDataRetention: false },logger: { warn: vi.fn(), info: vi.fn(), debug: vi.fn() },};const document: any = {};diff --git a/apps/api/src/scraper/scrapeURL/transformers/audio.ts b/apps/api/src/scraper/scrapeURL/transformers/audio.tsindex a8e9f5bc9..76fd0681e 100644--- a/apps/api/src/scraper/scrapeURL/transformers/audio.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/audio.ts@@ -53,6 +53,13 @@ export async function fetchAudio(return document;}+ if (meta.internalOptions.zeroDataRetention) {+ document.warning =+ "Audio format is not supported with zero data retention." ++ (document.warning ? " " + document.warning : "");+ return document;+ }+if (!config.AVGRAB_SERVICE_URL) {meta.logger.warn("AVGRAB_SERVICE_URL is not configured");document.warning =diff --git a/apps/api/src/scraper/scrapeURL/transformers/product.ts b/apps/api/src/scraper/scrapeURL/transformers/product.tsindex f74537349..751eabae4 100644--- a/apps/api/src/scraper/scrapeURL/transformers/product.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/product.ts@@ -11,6 +11,13 @@ export async function fetchProduct(return document;}+ if (meta.internalOptions.zeroDataRetention) {+ document.warning =+ "Product format is not supported with zero data retention." ++ (document.warning ? " " + document.warning : "");+ return document;+ }+if (!config.PRODUCT_EXTRACTION_SERVICE_URL) {meta.logger.warn("PRODUCT_EXTRACTION_SERVICE_URL is not configured");document.warning =diff --git a/apps/api/src/scraper/scrapeURL/transformers/video.ts b/apps/api/src/scraper/scrapeURL/transformers/video.tsindex 8a2d9d675..3caa1b133 100644--- a/apps/api/src/scraper/scrapeURL/transformers/video.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/video.ts@@ -211,6 +211,13 @@ export async function fetchVideo(return document;}+ if (meta.internalOptions.zeroDataRetention) {+ document.warning =+ "Video format is not supported with zero data retention." ++ (document.warning ? " " + document.warning : "");+ return document;+ }+if (!config.AVGRAB_SERVICE_URL) {meta.logger.warn("AVGRAB_SERVICE_URL is not configured");document.warning =