Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix enhanced using basic (#3845)
rowinsg5f45da6cd113
diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.tsindex 6899a3ad1..a287bf11c 100644--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts@@ -1322,6 +1322,55 @@ describe("Scrape tests", () => {scrapeTimeout * 2,);+ // Regression: an explicit stealth/enhanced proxy must still use stealth+ // even when another feature flag (e.g. actions) is requested. The engine+ // picker used to drop the negative-quality stealth engines via the quality+ // filter, so a request with a non-stealth flag would silently fall back to+ // a basic proxy.+ it.concurrent(+ "enhanced uses stealth alongside other feature flags",+ async () => {+ const res = await scrape(+ {+ url: base,+ proxy: "enhanced",+ actions: [+ {+ type: "wait",+ milliseconds: 500,+ },+ ],+ },+ identity,+ );++ expect(res.metadata.proxyUsed).toBe("stealth");+ },+ scrapeTimeout * 2,+ );++ it.concurrent(+ "stealth uses stealth alongside other feature flags",+ async () => {+ const res = await scrape(+ {+ url: base,+ proxy: "stealth",+ actions: [+ {+ type: "wait",+ milliseconds: 500,+ },+ ],+ },+ identity,+ );++ expect(res.metadata.proxyUsed).toBe("stealth");+ },+ scrapeTimeout * 2,+ );+// TODO: flaky// it.concurrent("auto works properly on 'stealth' site (faked for reliabile testing)", async () => {// const res = await scrape({diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 2c1818ae2..89ace87f3 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -649,6 +649,22 @@ export async function buildFallbackList(meta: Meta): Promise<}}+ // When stealth proxy is explicitly requested (proxy: "stealth" | "enhanced"),+ // restrict the fallback list to engines that actually support it. Stealth+ // engines all carry negative quality, so without this the quality filter+ // below would drop them in favor of a regular positive-quality engine,+ // silently ignoring the user's request and never attempting stealth.+ // The guard keeps the original list if no stealth-capable engine qualified+ // (e.g. self-hosted without fire-engine) so scrapes don't break entirely.+ if (meta.featureFlags.has("stealthProxy")) {+ const stealthCapable = selectedEngines.filter(+ x => !x.unsupportedFeatures.has("stealthProxy"),+ );+ if (stealthCapable.length > 0) {+ selectedEngines = stealthCapable;+ }+ }+if (selectedEngines.some(x => engineOptions[x.engine].quality > 0 && !x.engine.startsWith("index"),