Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(scrape): pass CDP cookies to audio downloads (#3472)

Gergő Móricz6b2651d98300
diff --git a/apps/api/src/__tests__/snips/v2/audio-routing.test.ts b/apps/api/src/__tests__/snips/v2/audio-routing.test.ts
index 0a3f04b7b..834d06bab 100644
--- a/apps/api/src/__tests__/snips/v2/audio-routing.test.ts
+++ b/apps/api/src/__tests__/snips/v2/audio-routing.test.ts
@@ -1,8 +1,7 @@
/**
* Unit test for audio-format engine routing via buildFallbackList.
- * Verifies that requesting audio format selects only index + tlsclient
- * engines (chrome-cdp and others are excluded) and that non-audio
- * requests still route through chrome-cdp as primary.
+ * Verifies that requesting audio format routes through chrome-cdp before
+ * audio postprocessing so browser cookies are available for avgrab.
*/
// Avoid jest ESM-parse issues on transitive `uuid` import when pulling in engines.
@@ -61,22 +60,26 @@ describe("Audio format engine routing (buildFallbackList)", () => {
},
}) as any;
- it("routes audio format to index then tlsclient only", async () => {
+ it("routes audio format to chrome-cdp before tlsclient", async () => {
const fallback = await buildFallbackList(buildStubMeta(["audio"]));
const engines = fallback.map(f => f.engine);
- // Cache-first (index, quality 1000), then tlsclient (quality 10).
- // index;documents and tlsclient;stealth drop out via the positive-quality filter.
- expect(engines).toEqual(["index", "fire-engine;tlsclient"]);
+ expect(engines).toEqual([
+ "fire-engine;chrome-cdp",
+ "fire-engine(retry);chrome-cdp",
+ "fire-engine;tlsclient",
+ ]);
});
- it("excludes chrome-cdp engines when audio format is requested", async () => {
+ it("excludes index and non-browser engines when audio format is requested", async () => {
const fallback = await buildFallbackList(buildStubMeta(["audio"]));
const engines = fallback.map(f => f.engine);
- expect(engines).not.toContain("fire-engine;chrome-cdp");
+ expect(engines).toContain("fire-engine;chrome-cdp");
+ expect(engines).toContain("fire-engine(retry);chrome-cdp");
+ expect(engines).not.toContain("index");
+ expect(engines).not.toContain("index;documents");
expect(engines).not.toContain("fire-engine;chrome-cdp;stealth");
- expect(engines).not.toContain("fire-engine(retry);chrome-cdp");
expect(engines).not.toContain("fire-engine(retry);chrome-cdp;stealth");
expect(engines).not.toContain("fetch");
});
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
index eb3babd35..a28f5ddd2 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
@@ -19,6 +19,13 @@ import { fireEngineURL } from "./scrape";
import { getDocFromGCS } from "../../../../lib/gcs-jobs";
import { Meta } from "../..";
+const browserCookieSchema = z
+ .object({
+ name: z.string(),
+ value: z.string(),
+ })
+ .passthrough();
+
const successSchema = z.object({
jobId: z.string(),
state: z.literal("completed"),
@@ -82,6 +89,15 @@ const successSchema = z.object({
link: z.string(),
}),
}),
+ z.object({
+ idx: z.number(),
+ type: z.literal("getCookies"),
+ result: z
+ .object({
+ cookies: browserCookieSchema.array(),
+ })
+ .passthrough(),
+ }),
])
.array()
.optional(),
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
index 862ff643b..3481dd13b 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -264,6 +264,7 @@ export async function scrapeURLWithFireEngineChromeCDP(
"engine.team_id": meta.internalOptions.teamId,
});
const hasBranding = hasFormatOfType(meta.options.formats, "branding");
+ const hasAudio = hasFormatOfType(meta.options.formats, "audio");
const defaultWait = hasBranding ? BRANDING_DEFAULT_WAIT_MS : 0;
const effectiveWait =
meta.options.waitFor != null && meta.options.waitFor !== 0
@@ -316,6 +317,14 @@ export async function scrapeURLWithFireEngineChromeCDP(
},
]
: []),
+ ...(hasAudio
+ ? ([
+ {
+ type: "getCookies",
+ metadata: { __firecrawl_internal: true },
+ },
+ ] as unknown as InternalAction[])
+ : []),
];
const totalWait = actions.reduce(
@@ -421,6 +430,9 @@ export async function scrapeURLWithFireEngineChromeCDP(
};
}
});
+ const audioCookies = (response.actionResults ?? [])
+ .filter(x => x.type === "getCookies")
+ .flatMap(x => x.result.cookies);
return {
url: response.url ?? meta.url,
@@ -451,6 +463,7 @@ export async function scrapeURLWithFireEngineChromeCDP(
proxyUsed: response.usedMobileProxy ? "stealth" : "basic",
youtubeTranscriptContent: response.youtubeTranscriptContent,
timezone: response.timezone,
+ ...(hasAudio ? { audioCookies } : {}),
};
});
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
index f1483edf2..0a5773a58 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
@@ -19,6 +19,14 @@ import {
import { Meta } from "../..";
import { config } from "../../../../config";
+
+const browserCookieSchema = z
+ .object({
+ name: z.string(),
+ value: z.string(),
+ })
+ .passthrough();
+
export type FireEngineScrapeRequestCommon = {
url: string;
scrapeId?: string;
@@ -124,6 +132,15 @@ const successSchema = z.object({
link: z.string(),
}),
}),
+ z.object({
+ idx: z.number(),
+ type: z.literal("getCookies"),
+ result: z
+ .object({
+ cookies: browserCookieSchema.array(),
+ })
+ .passthrough(),
+ }),
])
.array()
.optional(),
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index a053c2fc5..bcb71c34b 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -1,6 +1,6 @@
import { ScrapeActionContent } from "../../../lib/entities";
import { config } from "../../../config";
-import { Meta } from "..";
+import type { BrowserCookie, Meta } from "..";
import { documentMaxReasonableTime, scrapeDocument } from "./document";
import {
fireEngineMaxReasonableTime,
@@ -155,6 +155,7 @@ export type EngineScrapeResult = {
youtubeTranscriptContent?: any;
postprocessorsUsed?: string[];
+ audioCookies?: BrowserCookie[];
proxyUsed: "basic" | "stealth";
timezone?: string;
@@ -222,7 +223,7 @@ const engineOptions: {
"screenshot@fullScreen": true,
pdf: false,
document: false,
- audio: true,
+ audio: false,
atsv: false,
mobile: true,
location: true,
@@ -242,7 +243,7 @@ const engineOptions: {
"screenshot@fullScreen": true, // through actions transform
pdf: false,
document: false,
- audio: false,
+ audio: true,
atsv: false,
location: true,
mobile: true,
@@ -262,7 +263,7 @@ const engineOptions: {
"screenshot@fullScreen": true, // through actions transform
pdf: false,
document: false,
- audio: false,
+ audio: true,
atsv: false,
location: true,
mobile: true,
@@ -282,7 +283,7 @@ const engineOptions: {
"screenshot@fullScreen": true,
pdf: true,
document: true,
- audio: true,
+ audio: false,
atsv: false,
location: true,
mobile: true,
@@ -302,7 +303,7 @@ const engineOptions: {
"screenshot@fullScreen": true, // through actions transform
pdf: false,
document: false,
- audio: false,
+ audio: true,
atsv: false,
location: true,
mobile: true,
@@ -322,7 +323,7 @@ const engineOptions: {
"screenshot@fullScreen": true, // through actions transform
pdf: false,
document: false,
- audio: false,
+ audio: true,
atsv: false,
location: true,
mobile: true,
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 66dc30e90..eac798f5e 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -98,6 +98,18 @@ export type ScrapeUrlResponse =
error: any;
};
+export type BrowserCookie = {
+ name: string;
+ value: string;
+ domain?: string;
+ path?: string;
+ expires?: number;
+ httpOnly?: boolean;
+ secure?: boolean;
+ sameSite?: string;
+ [key: string]: unknown;
+};
+
export type Meta = {
id: string;
url: string;
@@ -141,6 +153,7 @@ export type Meta = {
costTracking: CostTracking;
winnerEngine?: Engine;
abortHandle?: NodeJS.Timeout;
+ audioCookies?: BrowserCookie[];
};
function buildFeatureFlags(
@@ -921,6 +934,9 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
meta.winnerEngine = result.engine;
let engineResult: EngineScrapeResult = result.result;
+ meta.audioCookies = (
+ engineResult as { audioCookies?: BrowserCookie[] }
+ ).audioCookies;
for (const postprocessor of postprocessors) {
if (
diff --git a/apps/api/src/scraper/scrapeURL/transformers/__tests__/audio.test.ts b/apps/api/src/scraper/scrapeURL/transformers/__tests__/audio.test.ts
index 8ad8fac45..f288c3f4c 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/__tests__/audio.test.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/__tests__/audio.test.ts
@@ -1,13 +1,36 @@
import { fetchAudio } from "../audio";
+import { config } from "../../../../config";
describe("fetchAudio lockdown guard", () => {
const originalFetch = global.fetch;
+ const originalAvgrabServiceUrl = config.AVGRAB_SERVICE_URL;
afterEach(() => {
global.fetch = originalFetch;
+ config.AVGRAB_SERVICE_URL = originalAvgrabServiceUrl;
jest.clearAllMocks();
});
+ function mockSuccessfulAvgrab() {
+ const fetchSpy = jest.fn(async (url: string, _init?: RequestInit) => {
+ if (url.endsWith("/supported-urls")) {
+ return {
+ ok: true,
+ json: async () => ({ regex: "https://example\\.com/audio" }),
+ };
+ }
+
+ return {
+ ok: true,
+ json: async () => ({ public_url: "https://storage.example/audio.mp3" }),
+ };
+ });
+
+ global.fetch = fetchSpy as any;
+ config.AVGRAB_SERVICE_URL = "https://avgrab.example";
+ return fetchSpy;
+ }
+
it("does not issue any fetch when lockdown is true, even if audio format is requested", async () => {
const fetchSpy = jest.fn();
global.fetch = fetchSpy as any;
@@ -47,4 +70,68 @@ describe("fetchAudio lockdown guard", () => {
expect(fetchSpy).not.toHaveBeenCalled();
});
+
+ it("forwards browser cookies to avgrab when audio cookies are available", async () => {
+ const fetchSpy = mockSuccessfulAvgrab();
+ const cookies = [
+ {
+ name: "VISITOR_INFO1_LIVE",
+ value: "visitor",
+ domain: ".youtube.com",
+ path: "/",
+ secure: true,
+ httpOnly: true,
+ },
+ ];
+
+ const meta: any = {
+ url: "https://example.com/audio",
+ audioCookies: cookies,
+ options: {
+ lockdown: false,
+ formats: [{ type: "audio" }],
+ },
+ logger: { warn: jest.fn(), info: jest.fn(), debug: jest.fn() },
+ };
+ const document: any = {};
+
+ await fetchAudio(meta, document);
+
+ const downloadCall = fetchSpy.mock.calls.find(([url]) =>
+ String(url).endsWith("/download"),
+ );
+ expect(downloadCall).toBeDefined();
+ const body = downloadCall![1]?.body;
+ expect(typeof body).toBe("string");
+ expect(JSON.parse(body as string)).toEqual({
+ url: "https://example.com/audio",
+ cookies,
+ });
+ });
+
+ it("omits cookies from the avgrab request when no audio cookies are available", async () => {
+ const fetchSpy = mockSuccessfulAvgrab();
+
+ const meta: any = {
+ url: "https://example.com/audio",
+ options: {
+ lockdown: false,
+ formats: [{ type: "audio" }],
+ },
+ logger: { warn: jest.fn(), info: jest.fn(), debug: jest.fn() },
+ };
+ const document: any = {};
+
+ await fetchAudio(meta, document);
+
+ const downloadCall = fetchSpy.mock.calls.find(([url]) =>
+ String(url).endsWith("/download"),
+ );
+ expect(downloadCall).toBeDefined();
+ const body = downloadCall![1]?.body;
+ expect(typeof body).toBe("string");
+ expect(JSON.parse(body as string)).toEqual({
+ url: "https://example.com/audio",
+ });
+ });
});
diff --git a/apps/api/src/scraper/scrapeURL/transformers/audio.ts b/apps/api/src/scraper/scrapeURL/transformers/audio.ts
index 6076bd178..046078cb6 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/audio.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/audio.ts
@@ -61,10 +61,17 @@ export async function fetchAudio(
throw new AudioUnsupportedUrlError();
}
+ const requestBody = {
+ url: meta.url,
+ ...(meta.audioCookies && meta.audioCookies.length > 0
+ ? { cookies: meta.audioCookies }
+ : {}),
+ };
+
const response = await fetch(`${config.AVGRAB_SERVICE_URL}/download`, {
method: "POST",
headers: { "Content-Type": "application/json" },
- body: JSON.stringify({ url: meta.url }),
+ body: JSON.stringify(requestBody),
});
if (!response.ok) {