Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
drop tlsclient stealth too (#4213)
rowinsg67f647fa25be
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 5ac762106..7160bfc4d 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -703,7 +703,21 @@ export async function buildFallbackList(meta: Meta): Promise<!meta.featureFlags.has("audio") &&!meta.featureFlags.has("video")) {- for (const engine of ["fire-engine;tlsclient", "fetch"] as Engine[]) {+ // The sort-time quality boost below cannot resurrect an engine that was+ // spliced out here, so dropping tlsclient unconditionally would silently+ // neuter the engpicker opt-in on exactly the TlsClientOk domains where+ // engpicker measured its output to match chrome-cdp's. Keep both variants+ // when that verdict is in hand; regular scrapes never set the flag, so they+ // drop tlsclient as intended.+ const enginesToDrop: Engine[] = ["fetch"];+ if (!shouldPrioritizeTlsClient) {+ enginesToDrop.push(+ "fire-engine;tlsclient",+ "fire-engine;tlsclient;stealth",+ );+ }++ for (const engine of enginesToDrop) {const index = _engines.indexOf(engine);if (index !== -1) {_engines.splice(index, 1);