Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(scrape): pass CDP cookies to audio downloads (#3472)
Gergő Móricz6b2651d98300
diff --git a/apps/api/src/__tests__/snips/v2/audio-routing.test.ts b/apps/api/src/__tests__/snips/v2/audio-routing.test.tsindex 0a3f04b7b..834d06bab 100644--- a/apps/api/src/__tests__/snips/v2/audio-routing.test.ts+++ b/apps/api/src/__tests__/snips/v2/audio-routing.test.ts@@ -1,8 +1,7 @@/*** Unit test for audio-format engine routing via buildFallbackList.- * Verifies that requesting audio format selects only index + tlsclient- * engines (chrome-cdp and others are excluded) and that non-audio- * requests still route through chrome-cdp as primary.+ * Verifies that requesting audio format routes through chrome-cdp before+ * audio postprocessing so browser cookies are available for avgrab.*/// Avoid jest ESM-parse issues on transitive `uuid` import when pulling in engines.@@ -61,22 +60,26 @@ describe("Audio format engine routing (buildFallbackList)", () => {},}) as any;- it("routes audio format to index then tlsclient only", async () => {+ it("routes audio format to chrome-cdp before tlsclient", async () => {const fallback = await buildFallbackList(buildStubMeta(["audio"]));const engines = fallback.map(f => f.engine);- // Cache-first (index, quality 1000), then tlsclient (quality 10).- // index;documents and tlsclient;stealth drop out via the positive-quality filter.- expect(engines).toEqual(["index", "fire-engine;tlsclient"]);+ expect(engines).toEqual([+ "fire-engine;chrome-cdp",+ "fire-engine(retry);chrome-cdp",+ "fire-engine;tlsclient",+ ]);});- it("excludes chrome-cdp engines when audio format is requested", async () => {+ it("excludes index and non-browser engines when audio format is requested", async () => {const fallback = await buildFallbackList(buildStubMeta(["audio"]));const engines = fallback.map(f => f.engine);- expect(engines).not.toContain("fire-engine;chrome-cdp");+ expect(engines).toContain("fire-engine;chrome-cdp");+ expect(engines).toContain("fire-engine(retry);chrome-cdp");+ expect(engines).not.toContain("index");+ expect(engines).not.toContain("index;documents");expect(engines).not.toContain("fire-engine;chrome-cdp;stealth");- expect(engines).not.toContain("fire-engine(retry);chrome-cdp");expect(engines).not.toContain("fire-engine(retry);chrome-cdp;stealth");expect(engines).not.toContain("fetch");});diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.tsindex eb3babd35..a28f5ddd2 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts@@ -19,6 +19,13 @@ import { fireEngineURL } from "./scrape";import { getDocFromGCS } from "../../../../lib/gcs-jobs";import { Meta } from "../..";+const browserCookieSchema = z+ .object({+ name: z.string(),+ value: z.string(),+ })+ .passthrough();+const successSchema = z.object({jobId: z.string(),state: z.literal("completed"),@@ -82,6 +89,15 @@ const successSchema = z.object({link: z.string(),}),}),+ z.object({+ idx: z.number(),+ type: z.literal("getCookies"),+ result: z+ .object({+ cookies: browserCookieSchema.array(),+ })+ .passthrough(),+ }),]).array().optional(),diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex 862ff643b..3481dd13b 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -264,6 +264,7 @@ export async function scrapeURLWithFireEngineChromeCDP("engine.team_id": meta.internalOptions.teamId,});const hasBranding = hasFormatOfType(meta.options.formats, "branding");+ const hasAudio = hasFormatOfType(meta.options.formats, "audio");const defaultWait = hasBranding ? BRANDING_DEFAULT_WAIT_MS : 0;const effectiveWait =meta.options.waitFor != null && meta.options.waitFor !== 0@@ -316,6 +317,14 @@ export async function scrapeURLWithFireEngineChromeCDP(},]: []),+ ...(hasAudio+ ? ([+ {+ type: "getCookies",+ metadata: { __firecrawl_internal: true },+ },+ ] as unknown as InternalAction[])+ : []),];const totalWait = actions.reduce(@@ -421,6 +430,9 @@ export async function scrapeURLWithFireEngineChromeCDP(};}});+ const audioCookies = (response.actionResults ?? [])+ .filter(x => x.type === "getCookies")+ .flatMap(x => x.result.cookies);return {url: response.url ?? meta.url,@@ -451,6 +463,7 @@ export async function scrapeURLWithFireEngineChromeCDP(proxyUsed: response.usedMobileProxy ? "stealth" : "basic",youtubeTranscriptContent: response.youtubeTranscriptContent,timezone: response.timezone,+ ...(hasAudio ? { audioCookies } : {}),};});}diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.tsindex f1483edf2..0a5773a58 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts@@ -19,6 +19,14 @@ import {import { Meta } from "../..";import { config } from "../../../../config";++const browserCookieSchema = z+ .object({+ name: z.string(),+ value: z.string(),+ })+ .passthrough();+export type FireEngineScrapeRequestCommon = {url: string;scrapeId?: string;@@ -124,6 +132,15 @@ const successSchema = z.object({link: z.string(),}),}),+ z.object({+ idx: z.number(),+ type: z.literal("getCookies"),+ result: z+ .object({+ cookies: browserCookieSchema.array(),+ })+ .passthrough(),+ }),]).array().optional(),diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex a053c2fc5..bcb71c34b 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -1,6 +1,6 @@import { ScrapeActionContent } from "../../../lib/entities";import { config } from "../../../config";-import { Meta } from "..";+import type { BrowserCookie, Meta } from "..";import { documentMaxReasonableTime, scrapeDocument } from "./document";import {fireEngineMaxReasonableTime,@@ -155,6 +155,7 @@ export type EngineScrapeResult = {youtubeTranscriptContent?: any;postprocessorsUsed?: string[];+ audioCookies?: BrowserCookie[];proxyUsed: "basic" | "stealth";timezone?: string;@@ -222,7 +223,7 @@ const engineOptions: {"screenshot@fullScreen": true,pdf: false,document: false,- audio: true,+ audio: false,atsv: false,mobile: true,location: true,@@ -242,7 +243,7 @@ const engineOptions: {"screenshot@fullScreen": true, // through actions transformpdf: false,document: false,- audio: false,+ audio: true,atsv: false,location: true,mobile: true,@@ -262,7 +263,7 @@ const engineOptions: {"screenshot@fullScreen": true, // through actions transformpdf: false,document: false,- audio: false,+ audio: true,atsv: false,location: true,mobile: true,@@ -282,7 +283,7 @@ const engineOptions: {"screenshot@fullScreen": true,pdf: true,document: true,- audio: true,+ audio: false,atsv: false,location: true,mobile: true,@@ -302,7 +303,7 @@ const engineOptions: {"screenshot@fullScreen": true, // through actions transformpdf: false,document: false,- audio: false,+ audio: true,atsv: false,location: true,mobile: true,@@ -322,7 +323,7 @@ const engineOptions: {"screenshot@fullScreen": true, // through actions transformpdf: false,document: false,- audio: false,+ audio: true,atsv: false,location: true,mobile: true,diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 66dc30e90..eac798f5e 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -98,6 +98,18 @@ export type ScrapeUrlResponse =error: any;};+export type BrowserCookie = {+ name: string;+ value: string;+ domain?: string;+ path?: string;+ expires?: number;+ httpOnly?: boolean;+ secure?: boolean;+ sameSite?: string;+ [key: string]: unknown;+};+export type Meta = {id: string;url: string;@@ -141,6 +153,7 @@ export type Meta = {costTracking: CostTracking;winnerEngine?: Engine;abortHandle?: NodeJS.Timeout;+ audioCookies?: BrowserCookie[];};function buildFeatureFlags(@@ -921,6 +934,9 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {meta.winnerEngine = result.engine;let engineResult: EngineScrapeResult = result.result;+ meta.audioCookies = (+ engineResult as { audioCookies?: BrowserCookie[] }+ ).audioCookies;for (const postprocessor of postprocessors) {if (diff --git a/apps/api/src/scraper/scrapeURL/transformers/__tests__/audio.test.ts b/apps/api/src/scraper/scrapeURL/transformers/__tests__/audio.test.tsindex 8ad8fac45..f288c3f4c 100644--- a/apps/api/src/scraper/scrapeURL/transformers/__tests__/audio.test.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/__tests__/audio.test.ts@@ -1,13 +1,36 @@import { fetchAudio } from "../audio";+import { config } from "../../../../config";describe("fetchAudio lockdown guard", () => {const originalFetch = global.fetch;+ const originalAvgrabServiceUrl = config.AVGRAB_SERVICE_URL;afterEach(() => {global.fetch = originalFetch;+ config.AVGRAB_SERVICE_URL = originalAvgrabServiceUrl;jest.clearAllMocks();});+ function mockSuccessfulAvgrab() {+ const fetchSpy = jest.fn(async (url: string, _init?: RequestInit) => {+ if (url.endsWith("/supported-urls")) {+ return {+ ok: true,+ json: async () => ({ regex: "https://example\\.com/audio" }),+ };+ }++ return {+ ok: true,+ json: async () => ({ public_url: "https://storage.example/audio.mp3" }),+ };+ });++ global.fetch = fetchSpy as any;+ config.AVGRAB_SERVICE_URL = "https://avgrab.example";+ return fetchSpy;+ }+it("does not issue any fetch when lockdown is true, even if audio format is requested", async () => {const fetchSpy = jest.fn();global.fetch = fetchSpy as any;@@ -47,4 +70,68 @@ describe("fetchAudio lockdown guard", () => {expect(fetchSpy).not.toHaveBeenCalled();});++ it("forwards browser cookies to avgrab when audio cookies are available", async () => {+ const fetchSpy = mockSuccessfulAvgrab();+ const cookies = [+ {+ name: "VISITOR_INFO1_LIVE",+ value: "visitor",+ domain: ".youtube.com",+ path: "/",+ secure: true,+ httpOnly: true,+ },+ ];++ const meta: any = {+ url: "https://example.com/audio",+ audioCookies: cookies,+ options: {+ lockdown: false,+ formats: [{ type: "audio" }],+ },+ logger: { warn: jest.fn(), info: jest.fn(), debug: jest.fn() },+ };+ const document: any = {};++ await fetchAudio(meta, document);++ const downloadCall = fetchSpy.mock.calls.find(([url]) =>+ String(url).endsWith("/download"),+ );+ expect(downloadCall).toBeDefined();+ const body = downloadCall![1]?.body;+ expect(typeof body).toBe("string");+ expect(JSON.parse(body as string)).toEqual({+ url: "https://example.com/audio",+ cookies,+ });+ });++ it("omits cookies from the avgrab request when no audio cookies are available", async () => {+ const fetchSpy = mockSuccessfulAvgrab();++ const meta: any = {+ url: "https://example.com/audio",+ options: {+ lockdown: false,+ formats: [{ type: "audio" }],+ },+ logger: { warn: jest.fn(), info: jest.fn(), debug: jest.fn() },+ };+ const document: any = {};++ await fetchAudio(meta, document);++ const downloadCall = fetchSpy.mock.calls.find(([url]) =>+ String(url).endsWith("/download"),+ );+ expect(downloadCall).toBeDefined();+ const body = downloadCall![1]?.body;+ expect(typeof body).toBe("string");+ expect(JSON.parse(body as string)).toEqual({+ url: "https://example.com/audio",+ });+ });});diff --git a/apps/api/src/scraper/scrapeURL/transformers/audio.ts b/apps/api/src/scraper/scrapeURL/transformers/audio.tsindex 6076bd178..046078cb6 100644--- a/apps/api/src/scraper/scrapeURL/transformers/audio.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/audio.ts@@ -61,10 +61,17 @@ export async function fetchAudio(throw new AudioUnsupportedUrlError();}+ const requestBody = {+ url: meta.url,+ ...(meta.audioCookies && meta.audioCookies.length > 0+ ? { cookies: meta.audioCookies }+ : {}),+ };+const response = await fetch(`${config.AVGRAB_SERVICE_URL}/download`, {method: "POST",headers: { "Content-Type": "application/json" },- body: JSON.stringify({ url: meta.url }),+ body: JSON.stringify(requestBody),});if (!response.ok) {