Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(scrapeURL): drop tlsclient/fetch from the waterfall when fire-engine is present (#4168)
Gergő Móricz4ed1a954eadb
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 9660a4584..5ac762106 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -688,6 +688,29 @@ export async function buildFallbackList(meta: Meta): Promise<}}+ // When fire-engine is available, drop tlsclient and fetch from the general+ // waterfall: once chrome-cdp (and its retry) fail, degrading to a plain+ // HTTP client tends to produce bot-walled or otherwise low-quality content,+ // so we'd rather fail the scrape outright. They stay reachable when the+ // request asks for them: fastMode/atsv set feature flags that chrome-cdp+ // can't satisfy (and are handled here), audio/video keep tlsclient as the+ // avgrab fallback behind chrome-cdp, and forceEngine bypasses _engines+ // entirely, reading straight from internalOptions.+ if (+ useFireEngine &&+ !meta.featureFlags.has("useFastMode") &&+ !meta.featureFlags.has("atsv") &&+ !meta.featureFlags.has("audio") &&+ !meta.featureFlags.has("video")+ ) {+ for (const engine of ["fire-engine;tlsclient", "fetch"] as Engine[]) {+ const index = _engines.indexOf(engine);+ if (index !== -1) {+ _engines.splice(index, 1);+ }+ }+ }+if (!isWikimediaUrl(meta.url) || Math.random() >= 0.5) {const wikiIndex = _engines.indexOf("wikipedia");if (wikiIndex !== -1) {