Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(scrape): skip YouTube transcript script for audio-only scrapes (#3473)

firecrawl-spring[bot]6944e55a2bd5
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
index 3481dd13b..6712d9614 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -265,6 +265,7 @@ export async function scrapeURLWithFireEngineChromeCDP(
});
const hasBranding = hasFormatOfType(meta.options.formats, "branding");
const hasAudio = hasFormatOfType(meta.options.formats, "audio");
+ const onlyAudio = !!hasAudio && (meta.options.formats?.length ?? 0) === 1;
const defaultWait = hasBranding ? BRANDING_DEFAULT_WAIT_MS : 0;
const effectiveWait =
meta.options.waitFor != null && meta.options.waitFor !== 0
@@ -363,6 +364,7 @@ export async function scrapeURLWithFireEngineChromeCDP(
meta.internalOptions.saveScrapeResultToGCS,
zeroDataRetention: meta.internalOptions.zeroDataRetention,
...(shouldAllowMedia ? { blockMedia: false } : {}),
+ ...(onlyAudio ? { skipYouTubeTranscript: true } : {}),
persistentStorage: meta.options.profile
? {
uniqueId: `${createHash("sha256").update(meta.internalOptions.teamId).digest("hex").slice(0, 16)}_${meta.options.profile.name}`,
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
index 0a5773a58..03cdd1d29 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
@@ -63,6 +63,7 @@ export type FireEngineScrapeRequestChromeCDP = {
mobile?: boolean;
disableSmartWaitCache?: boolean;
persistentStorage?: { uniqueId: string };
+ skipYouTubeTranscript?: boolean;
};
export type FireEngineScrapeRequestTLSClient = {