Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): refuse audio, video and product formats under zero data retention (#5423)

Gergő Móriczcccc2a6e554b
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml
index 193152b2a..f6cef5367 100644
--- a/.github/workflows/test-server.yml
+++ b/.github/workflows/test-server.yml
@@ -333,6 +333,17 @@ jobs:
--reporter=default
working-directory: apps/api
+ - name: Run audio, video and product format zero data retention regression tests
+ if: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'
+ run: >-
+ pnpm exec vitest run
+ src/scraper/scrapeURL/transformers/__tests__/audio.test.ts
+ src/scraper/scrapeURL/transformers/__tests__/video.test.ts
+ src/scraper/scrapeURL/transformers/__tests__/product.test.ts
+ src/lib/scrape-billing.test.ts
+ --reporter=default
+ working-directory: apps/api
+
- name: Run country check route regression tests
if: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'
run: pnpm exec vitest run src/__tests__/routes/country-check.routes.test.ts --reporter=default
diff --git a/apps/api/src/__tests__/snips/v2/zdr.test.ts b/apps/api/src/__tests__/snips/v2/zdr.test.ts
index ecb18b887..81a2749eb 100644
--- a/apps/api/src/__tests__/snips/v2/zdr.test.ts
+++ b/apps/api/src/__tests__/snips/v2/zdr.test.ts
@@ -7,8 +7,17 @@ import {
zdrcleaner,
idmux,
searchRaw,
+ Identity,
} from "./lib";
-import { describeIf, itIf, TEST_PRODUCTION, scrapeTimeout } from "../lib";
+import {
+ describeIf,
+ itIf,
+ concurrentIf,
+ TEST_PRODUCTION,
+ TEST_SUITE_WEBSITE,
+ HAS_PRODUCT_SERVICE,
+ scrapeTimeout,
+} from "../lib";
import {
getLogs,
expectScrapeIsCleanedUp,
@@ -224,4 +233,145 @@ describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {
expect(response.statusCode).toBe(200);
expect(response.body.success).toBe(true);
}, 60000);
+
+ describe.each(scopes)("%s audio, video and product formats", scope => {
+ let identity: Identity;
+
+ beforeAll(async () => {
+ identity = await idmux({
+ name: `zdr/${scope}/formats`,
+ credits: 10000,
+ flags: {
+ scrapeZDR: scope === "Team-scoped" ? "forced" : "allowed",
+ },
+ });
+ }, scrapeTimeout);
+
+ const zeroDataRetention = scope === "Request-scoped" ? true : undefined;
+
+ it.concurrent(
+ "refuses the audio format without attempting the download",
+ async () => {
+ // Not a supported media URL: a download attempt would fail the scrape.
+ const response = await scrape(
+ {
+ url: "https://firecrawl.dev",
+ formats: ["markdown", "audio"],
+ zeroDataRetention,
+ },
+ identity,
+ );
+
+ expect(response.markdown).toBeDefined();
+ expect(response.audio).toBeUndefined();
+ expect(response.warning).toContain(
+ "Audio format is not supported with zero data retention.",
+ );
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "refuses the video format without attempting the download",
+ async () => {
+ const response = await scrape(
+ {
+ url: "https://firecrawl.dev",
+ formats: ["markdown", "video"],
+ zeroDataRetention,
+ },
+ identity,
+ );
+
+ expect(response.markdown).toBeDefined();
+ expect(response.video).toBeUndefined();
+ expect(response.videos).toBeUndefined();
+ expect(response.warning).toContain(
+ "Video format is not supported with zero data retention.",
+ );
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "refuses the product format",
+ async () => {
+ const response = await scrape(
+ {
+ url: `${TEST_SUITE_WEBSITE}/product`,
+ formats: ["markdown", { type: "product" }],
+ zeroDataRetention,
+ },
+ identity,
+ );
+
+ expect(response.markdown).toBeDefined();
+ expect(response.product).toBeUndefined();
+ expect(response.warning).toContain(
+ "Product format is not supported with zero data retention.",
+ );
+ },
+ scrapeTimeout,
+ );
+ });
+
+ describe("audio, video and product formats without zero data retention on the request", () => {
+ let identity: Identity;
+
+ beforeAll(async () => {
+ identity = await idmux({
+ name: "zdr/allowed/formats-without-zdr",
+ credits: 10000,
+ flags: { scrapeZDR: "allowed" },
+ });
+ }, scrapeTimeout);
+
+ it.concurrent(
+ "still returns audio",
+ async () => {
+ const response = await scrape(
+ {
+ url: "https://www.youtube.com/watch?v=dQw4w9WgXcQ",
+ formats: ["audio"],
+ },
+ identity,
+ );
+
+ expect(response.audio).toMatch(/^https:\/\//);
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "still returns video",
+ async () => {
+ const response = await scrape(
+ {
+ url: "https://www.youtube.com/watch?v=dQw4w9WgXcQ",
+ formats: ["video"],
+ },
+ identity,
+ );
+
+ expect(response.video).toMatch(/^https:\/\//);
+ },
+ scrapeTimeout * 2,
+ );
+
+ concurrentIf(HAS_PRODUCT_SERVICE)(
+ "still returns the product",
+ async () => {
+ const response = await scrape(
+ {
+ url: `${TEST_SUITE_WEBSITE}/product`,
+ formats: [{ type: "product" }],
+ },
+ identity,
+ );
+
+ expect(response.product?.title).toBe("Firecrawl Test Widget");
+ },
+ scrapeTimeout,
+ );
+ });
});
diff --git a/apps/api/src/lib/scrape-billing.test.ts b/apps/api/src/lib/scrape-billing.test.ts
index fd25cebc4..46b3f1648 100644
--- a/apps/api/src/lib/scrape-billing.test.ts
+++ b/apps/api/src/lib/scrape-billing.test.ts
@@ -480,6 +480,16 @@ describe("calculateCreditsToBeBilled — zero data retention", () => {
it("is not added on top of lockdown", async () => {
expect(await bill({ lockdown: true }, true)).toBe(5);
});
+
+ it.each(["audio", "video"])(
+ "drops the %s surcharge it refuses, but not under lockdown",
+ async type => {
+ const media = { formats: [{ type }] };
+ expect(await bill(media, false)).toBe(5);
+ expect(await bill(media, true)).toBe(2);
+ expect(await bill({ ...media, lockdown: true }, true)).toBe(9);
+ },
+ );
});
describe("calculateThreatScanCredits", () => {
diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.ts
index 7df98515e..585c54dee 100644
--- a/apps/api/src/lib/scrape-billing.ts
+++ b/apps/api/src/lib/scrape-billing.ts
@@ -216,11 +216,16 @@ export async function calculateCreditsToBeBilled(
creditsToBeBilled += 4;
}
- if (hasFormatOfType(options.formats, "audio")) {
+ // Not billed when zero data retention refuses them (transformers/audio.ts,
+ // video.ts). Lockdown skips the fetch before that check and keeps the surcharge.
+ const mediaFormatsRefused =
+ !!internalOptions.zeroDataRetention && !options.lockdown;
+
+ if (hasFormatOfType(options.formats, "audio") && !mediaFormatsRefused) {
creditsToBeBilled += 4;
}
- if (hasFormatOfType(options.formats, "video")) {
+ if (hasFormatOfType(options.formats, "video") && !mediaFormatsRefused) {
creditsToBeBilled += 4;
}
diff --git a/apps/api/src/scraper/scrapeURL/transformers/__tests__/audio.test.ts b/apps/api/src/scraper/scrapeURL/transformers/__tests__/audio.test.ts
index ec6591116..3640f0479 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/__tests__/audio.test.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/__tests__/audio.test.ts
@@ -53,6 +53,29 @@ describe("fetchAudio lockdown guard", () => {
expect(result.audio).toBeUndefined();
});
+ it("refuses the audio format under zero data retention without calling avgrab", async () => {
+ const fetchSpy = mockSuccessfulAvgrab();
+
+ const meta: any = {
+ url: "https://example.com/audio",
+ options: {
+ lockdown: false,
+ formats: [{ type: "audio" }],
+ },
+ internalOptions: { zeroDataRetention: true },
+ logger: { warn: vi.fn(), info: vi.fn(), debug: vi.fn() },
+ };
+ const document: any = {};
+
+ const result = await fetchAudio(meta, document);
+
+ expect(fetchSpy).not.toHaveBeenCalled();
+ expect(result.audio).toBeUndefined();
+ expect(result.warning).toBe(
+ "Audio format is not supported with zero data retention.",
+ );
+ });
+
it("returns early when audio format is not requested regardless of lockdown", async () => {
const fetchSpy = vi.fn();
global.fetch = fetchSpy as any;
@@ -92,6 +115,7 @@ describe("fetchAudio lockdown guard", () => {
lockdown: false,
formats: [{ type: "audio" }],
},
+ internalOptions: { zeroDataRetention: false },
logger: { warn: vi.fn(), info: vi.fn(), debug: vi.fn() },
};
const document: any = {};
@@ -140,6 +164,7 @@ describe("fetchAudio lockdown guard", () => {
lockdown: false,
formats: [{ type: "audio" }],
},
+ internalOptions: { zeroDataRetention: false },
logger: { warn: vi.fn(), info: vi.fn(), debug: vi.fn() },
};
@@ -175,6 +200,7 @@ describe("fetchAudio lockdown guard", () => {
lockdown: false,
formats: [{ type: "audio" }],
},
+ internalOptions: { zeroDataRetention: false },
logger: { warn: vi.fn(), info: vi.fn(), debug: vi.fn() },
};
@@ -195,6 +221,7 @@ describe("fetchAudio lockdown guard", () => {
lockdown: false,
formats: [{ type: "audio" }],
},
+ internalOptions: { zeroDataRetention: false },
logger: { warn: vi.fn(), info: vi.fn(), debug: vi.fn() },
};
const document: any = {};
diff --git a/apps/api/src/scraper/scrapeURL/transformers/__tests__/product.test.ts b/apps/api/src/scraper/scrapeURL/transformers/__tests__/product.test.ts
index b501884a2..75cc69cc2 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/__tests__/product.test.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/__tests__/product.test.ts
@@ -11,11 +11,15 @@ describe("fetchProduct", () => {
vi.clearAllMocks();
});
- function baseMeta(formats: any[] = [{ type: "product" }]) {
+ function baseMeta(
+ formats: any[] = [{ type: "product" }],
+ zeroDataRetention = false,
+ ) {
return {
url: "https://shop.test/p",
rewrittenUrl: "https://shop.test/p",
options: { formats },
+ internalOptions: { zeroDataRetention },
logger: { warn: vi.fn(), info: vi.fn(), debug: vi.fn(), error: vi.fn() },
} as any;
}
@@ -36,6 +40,27 @@ describe("fetchProduct", () => {
expect(out.warning).toMatch(/not available/i);
});
+ it("refuses the product format under zero data retention without calling the service", async () => {
+ config.PRODUCT_EXTRACTION_SERVICE_URL = "https://product.internal";
+ const fetchSpy = vi.fn();
+ global.fetch = fetchSpy as any;
+ const document: any = {
+ rawHtml: "<html>jsonld</html>",
+ metadata: { url: "https://shop.test/p" },
+ };
+
+ const out = await fetchProduct(
+ baseMeta([{ type: "product" }], true),
+ document,
+ );
+
+ expect(fetchSpy).not.toHaveBeenCalled();
+ expect(out.product).toBeUndefined();
+ expect(out.warning).toBe(
+ "Product format is not supported with zero data retention.",
+ );
+ });
+
it("posts rawHtml to the service and sets document.product", async () => {
config.PRODUCT_EXTRACTION_SERVICE_URL = "https://product.internal";
const product = { title: "X", url: "https://shop.test/p", variants: [] };
diff --git a/apps/api/src/scraper/scrapeURL/transformers/__tests__/video.test.ts b/apps/api/src/scraper/scrapeURL/transformers/__tests__/video.test.ts
index a000ee9c8..6ca7a27d7 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/__tests__/video.test.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/__tests__/video.test.ts
@@ -62,6 +62,7 @@ describe("fetchVideo", () => {
lockdown: false,
formats: [{ type: "video" }],
},
+ internalOptions: { zeroDataRetention: false },
logger: { warn: vi.fn(), info: vi.fn(), debug: vi.fn() },
};
const document: any = { rawHtml: "<html></html>" };
@@ -104,6 +105,36 @@ describe("fetchVideo", () => {
expect(result.video).toBeUndefined();
});
+ it.each([
+ "https://www.youtube.com/watch?v=abc123",
+ "https://example.com/product",
+ ])(
+ "refuses the video format under zero data retention without calling avgrab (%s)",
+ async url => {
+ const fetchSpy = mockSuccessfulAvgrab();
+
+ const meta: any = {
+ url,
+ options: {
+ lockdown: false,
+ formats: [{ type: "video" }],
+ },
+ internalOptions: { zeroDataRetention: true },
+ logger: { warn: vi.fn(), info: vi.fn(), debug: vi.fn() },
+ };
+ const document: any = { rawHtml: "<html></html>" };
+
+ const result = await fetchVideo(meta, document);
+
+ expect(fetchSpy).not.toHaveBeenCalled();
+ expect(result.video).toBeUndefined();
+ expect(result.videos).toBeUndefined();
+ expect(result.warning).toBe(
+ "Video format is not supported with zero data retention.",
+ );
+ },
+ );
+
it("returns early when video format is not requested regardless of lockdown", async () => {
const fetchSpy = vi.fn();
global.fetch = fetchSpy as any;
@@ -114,6 +145,7 @@ describe("fetchVideo", () => {
lockdown: false,
formats: [{ type: "markdown" }],
},
+ internalOptions: { zeroDataRetention: false },
logger: { warn: vi.fn(), info: vi.fn(), debug: vi.fn() },
};
const document: any = { markdown: "cached" };
@@ -134,6 +166,7 @@ describe("fetchVideo", () => {
lockdown: false,
formats: [{ type: "video" }],
},
+ internalOptions: { zeroDataRetention: false },
logger: { warn: vi.fn(), info: vi.fn(), debug: vi.fn() },
};
const document: any = {};
@@ -167,6 +200,7 @@ describe("fetchVideo", () => {
lockdown: false,
formats: [{ type: "video" }],
},
+ internalOptions: { zeroDataRetention: false },
logger: { warn: vi.fn(), info: vi.fn(), debug: vi.fn() },
};
const document: any = {};
@@ -212,6 +246,7 @@ describe("fetchVideo", () => {
lockdown: false,
formats: [{ type: "video" }],
},
+ internalOptions: { zeroDataRetention: false },
logger: { warn: vi.fn(), info: vi.fn(), debug: vi.fn() },
};
const document: any = {};
@@ -253,6 +288,7 @@ describe("fetchVideo", () => {
lockdown: false,
formats: [{ type: "video" }],
},
+ internalOptions: { zeroDataRetention: false },
logger: { warn: vi.fn(), info: vi.fn(), debug: vi.fn() },
};
const document: any = {};
@@ -276,6 +312,7 @@ describe("fetchVideo", () => {
lockdown: false,
formats: [{ type: "video" }],
},
+ internalOptions: { zeroDataRetention: false },
logger: { warn: vi.fn(), info: vi.fn(), debug: vi.fn() },
};
const document: any = {};
@@ -302,6 +339,7 @@ describe("fetchVideo", () => {
lockdown: false,
formats: [{ type: "video" }],
},
+ internalOptions: { zeroDataRetention: false },
logger: { warn: vi.fn(), info: vi.fn(), debug: vi.fn() },
};
const document: any = {};
diff --git a/apps/api/src/scraper/scrapeURL/transformers/audio.ts b/apps/api/src/scraper/scrapeURL/transformers/audio.ts
index a8e9f5bc9..76fd0681e 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/audio.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/audio.ts
@@ -53,6 +53,13 @@ export async function fetchAudio(
return document;
}
+ if (meta.internalOptions.zeroDataRetention) {
+ document.warning =
+ "Audio format is not supported with zero data retention." +
+ (document.warning ? " " + document.warning : "");
+ return document;
+ }
+
if (!config.AVGRAB_SERVICE_URL) {
meta.logger.warn("AVGRAB_SERVICE_URL is not configured");
document.warning =
diff --git a/apps/api/src/scraper/scrapeURL/transformers/product.ts b/apps/api/src/scraper/scrapeURL/transformers/product.ts
index f74537349..751eabae4 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/product.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/product.ts
@@ -11,6 +11,13 @@ export async function fetchProduct(
return document;
}
+ if (meta.internalOptions.zeroDataRetention) {
+ document.warning =
+ "Product format is not supported with zero data retention." +
+ (document.warning ? " " + document.warning : "");
+ return document;
+ }
+
if (!config.PRODUCT_EXTRACTION_SERVICE_URL) {
meta.logger.warn("PRODUCT_EXTRACTION_SERVICE_URL is not configured");
document.warning =
diff --git a/apps/api/src/scraper/scrapeURL/transformers/video.ts b/apps/api/src/scraper/scrapeURL/transformers/video.ts
index 8a2d9d675..3caa1b133 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/video.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/video.ts
@@ -211,6 +211,13 @@ export async function fetchVideo(
return document;
}
+ if (meta.internalOptions.zeroDataRetention) {
+ document.warning =
+ "Video format is not supported with zero data retention." +
+ (document.warning ? " " + document.warning : "");
+ return document;
+ }
+
if (!config.AVGRAB_SERVICE_URL) {
meta.logger.warn("AVGRAB_SERVICE_URL is not configured");
document.warning =