Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(scrapeURL): drop tlsclient/fetch from the waterfall when fire-engine is present (#4168)

Gergő Móricz4ed1a954eadb
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index 9660a4584..5ac762106 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -688,6 +688,29 @@ export async function buildFallbackList(meta: Meta): Promise<
}
}
+ // When fire-engine is available, drop tlsclient and fetch from the general
+ // waterfall: once chrome-cdp (and its retry) fail, degrading to a plain
+ // HTTP client tends to produce bot-walled or otherwise low-quality content,
+ // so we'd rather fail the scrape outright. They stay reachable when the
+ // request asks for them: fastMode/atsv set feature flags that chrome-cdp
+ // can't satisfy (and are handled here), audio/video keep tlsclient as the
+ // avgrab fallback behind chrome-cdp, and forceEngine bypasses _engines
+ // entirely, reading straight from internalOptions.
+ if (
+ useFireEngine &&
+ !meta.featureFlags.has("useFastMode") &&
+ !meta.featureFlags.has("atsv") &&
+ !meta.featureFlags.has("audio") &&
+ !meta.featureFlags.has("video")
+ ) {
+ for (const engine of ["fire-engine;tlsclient", "fetch"] as Engine[]) {
+ const index = _engines.indexOf(engine);
+ if (index !== -1) {
+ _engines.splice(index, 1);
+ }
+ }
+ }
+
if (!isWikimediaUrl(meta.url) || Math.random() >= 0.5) {
const wikiIndex = _engines.indexOf("wikipedia");
if (wikiIndex !== -1) {