Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): keep page engines when stealth proxy is unavailable (#5713)
Abimael Martellb3868a988173
diff --git a/apps/api/src/__tests__/snips/v1/deprecation.test.ts b/apps/api/src/__tests__/snips/v1/deprecation.test.tsindex e3728f5dd..3a21b405c 100644--- a/apps/api/src/__tests__/snips/v1/deprecation.test.ts+++ b/apps/api/src/__tests__/snips/v1/deprecation.test.ts@@ -1,6 +1,6 @@import { describe, it, expect, beforeAll } from "vitest";import request from "supertest";-import { itIf, TEST_API_URL } from "../lib";+import { itIf, scrapeTimeout, TEST_API_URL } from "../lib";import { sunsetOf } from "../../../lib/deprecations";import {DEEP_RESEARCH_LIVE,@@ -140,7 +140,7 @@ describe("Deprecation warnings on legacy endpoints", () => {).toBe(true);expect(res.body.replacement).toBe("/v2/scrape");},- 60000,+ scrapeTimeout,);itV1(diff --git a/apps/api/src/__tests__/snips/v2/stealth-routing-fire-engine.test.ts b/apps/api/src/__tests__/snips/v2/stealth-routing-fire-engine.test.tsnew file mode 100644index 000000000..90a8ce6e9--- /dev/null+++ b/apps/api/src/__tests__/snips/v2/stealth-routing-fire-engine.test.ts@@ -0,0 +1,116 @@+/**+ * Unit test for stealth proxy engine routing via buildFallbackList with+ * fire-engine configured. These are the fallback lists fire-engine+ * deployments use, pinned so the self-hosted stealthProxy handling cannot+ * change them.+ */++describe("Stealth proxy routing with fire-engine (buildFallbackList)", () => {+ let buildFallbackList: typeof import("../../../scraper/scrapeURL/engines/index.js").buildFallbackList;++ const overrides: Record<string, string | undefined> = {+ FIRE_ENGINE_BETA_URL: "http://test-fire-engine",+ FIRE_EXCHANGE_URL: undefined,+ INDEX_DATABASE_URL: undefined,+ PLAYWRIGHT_MICROSERVICE_URL: undefined,+ };+ const original: Record<string, string | undefined> = {};++ const setEnv = (env: Record<string, string | undefined>) => {+ for (const [key, value] of Object.entries(env)) {+ if (value === undefined) {+ delete process.env[key];+ } else {+ process.env[key] = value;+ }+ }+ };++ beforeAll(async () => {+ for (const key of Object.keys(overrides)) {+ original[key] = process.env[key];+ }+ setEnv(overrides);++ // Re-import engines fresh so it reads the env vars set above at eval time.+ vi.resetModules();+ ({ buildFallbackList } = await import(+ "../../../scraper/scrapeURL/engines/index.js"+ ));+ });++ afterAll(() => {+ setEnv(original);+ });++ const engines = async (+ url: string,+ featureFlags: string[],+ proxy: "auto" | "stealth",+ ) =>+ (+ await buildFallbackList({+ id: "test",+ url,+ options: { formats: [], maxAge: 0, proxy },+ internalOptions: { teamId: "test" },+ featureFlags: new Set(featureFlags),+ mock: null,+ logger: {+ info: vi.fn(),+ warn: vi.fn(),+ debug: vi.fn(),+ error: vi.fn(),+ child: vi.fn().mockReturnThis(),+ },+ } as any)+ ).map(f => f.engine);++ const cdp = ["fire-engine;chrome-cdp", "fire-engine(retry);chrome-cdp"];+ const cdpStealth = [+ "fire-engine;chrome-cdp;stealth",+ "fire-engine(retry);chrome-cdp;stealth",+ ];++ it("routes a regular HTML scrape to chrome-cdp", async () => {+ expect(await engines("https://example.com/", [], "auto")).toEqual(cdp);+ });++ it("narrows an HTML scrape that escalated to stealthProxy to the stealth engines", async () => {+ expect(+ await engines("https://example.com/", ["stealthProxy"], "auto"),+ ).toEqual([...cdpStealth, "pdf", "document"]);+ });++ it("narrows an explicit stealth HTML scrape to the stealth engines", async () => {+ expect(+ await engines("https://example.com/", ["stealthProxy"], "stealth"),+ ).toEqual([...cdpStealth, "pdf", "document"]);+ });++ it("routes a PDF through the browser engines", async () => {+ expect(await engines("https://example.com/a.pdf", ["pdf"], "auto")).toEqual(+ [...cdp, ...cdpStealth],+ );+ });++ it("routes a PDF that escalated to stealthProxy through the stealth engines", async () => {+ expect(+ await engines(+ "https://example.com/a.pdf",+ ["pdf", "stealthProxy"],+ "auto",+ ),+ ).toEqual(cdpStealth);+ });++ it("routes an explicit stealth document through the stealth engines", async () => {+ expect(+ await engines(+ "https://example.com/a.docx",+ ["document", "stealthProxy"],+ "stealth",+ ),+ ).toEqual(cdpStealth);+ });+});diff --git a/apps/api/src/__tests__/snips/v2/stealth-routing.test.ts b/apps/api/src/__tests__/snips/v2/stealth-routing.test.tsnew file mode 100644index 000000000..d69d9bd49--- /dev/null+++ b/apps/api/src/__tests__/snips/v2/stealth-routing.test.ts@@ -0,0 +1,82 @@+/**+ * Unit test for stealth proxy engine routing via buildFallbackList.+ * Without fire-engine no engine fetches a page through a stealth proxy, so an+ * auto-proxy scrape that escalates to stealthProxy must keep the regular page+ * engines instead of narrowing to the file parsers.+ */++describe("Stealth proxy routing without fire-engine (buildFallbackList)", () => {+ let buildFallbackList: typeof import("../../../scraper/scrapeURL/engines/index.js").buildFallbackList;++ const overrides: Record<string, string | undefined> = {+ FIRE_ENGINE_BETA_URL: undefined,+ FIRE_EXCHANGE_URL: undefined,+ INDEX_DATABASE_URL: undefined,+ PLAYWRIGHT_MICROSERVICE_URL: "http://test-playwright",+ };+ const original: Record<string, string | undefined> = {};++ const setEnv = (env: Record<string, string | undefined>) => {+ for (const [key, value] of Object.entries(env)) {+ if (value === undefined) {+ delete process.env[key];+ } else {+ process.env[key] = value;+ }+ }+ };++ beforeAll(async () => {+ for (const key of Object.keys(overrides)) {+ original[key] = process.env[key];+ }+ setEnv(overrides);++ // Re-import engines fresh so it reads the env vars set above at eval time.+ vi.resetModules();+ ({ buildFallbackList } = await import(+ "../../../scraper/scrapeURL/engines/index.js"+ ));+ });++ afterAll(() => {+ setEnv(original);+ });++ const buildStubMeta = (featureFlags: string[]) =>+ ({+ id: "test",+ url: "https://example.com/",+ options: {+ formats: [],+ maxAge: 0,+ proxy: "auto",+ },+ internalOptions: { teamId: "test" },+ featureFlags: new Set(featureFlags),+ mock: null,+ logger: {+ info: vi.fn(),+ warn: vi.fn(),+ debug: vi.fn(),+ error: vi.fn(),+ child: vi.fn().mockReturnThis(),+ },+ }) as any;++ it("keeps the page engines when stealthProxy is added to an HTML scrape", async () => {+ const before = await buildFallbackList(buildStubMeta([]));+ const after = await buildFallbackList(buildStubMeta(["stealthProxy"]));++ expect(before.map(f => f.engine)).toEqual(["playwright", "fetch"]);+ expect(after.map(f => f.engine)).toEqual(["playwright", "fetch"]);+ });++ it("still routes PDFs to the pdf engine when stealthProxy is set", async () => {+ const fallback = await buildFallbackList(+ buildStubMeta(["pdf", "stealthProxy"]),+ );++ expect(fallback.map(f => f.engine)).toEqual(["pdf"]);+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 951d979f8..0829cbf12 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -942,10 +942,18 @@ export async function buildFallbackList(meta: Meta): Promise<}}- const prioritySum = [...meta.featureFlags].reduce(- (a, x) => a + featureFlagOptions[x].priority,- 0,- );+ // Without fire-engine (or its test mocks), no engine fetches a page through+ // a stealth proxy: only index lookups and the file parsers declare+ // stealthProxy support. Counting the flag would leave only the file parsers+ // above the threshold and in the stealth-only narrowing below, so an HTML+ // page that escalated to stealthProxy would fail on every engine.+ const scoresStealthProxy = useFireEngine || meta.mock !== null;+ const isScoredFlag = (flag: FeatureFlag) =>+ scoresStealthProxy || flag !== "stealthProxy";++ const prioritySum = [...meta.featureFlags]+ .filter(isScoredFlag)+ .reduce((a, x) => a + featureFlagOptions[x].priority, 0);const priorityThreshold = Math.floor(prioritySum / 2);let selectedEngines: {engine: Engine;@@ -984,10 +992,9 @@ export async function buildFallbackList(meta: Meta): Promise<).map(([k, _]) => k),]);- const supportScore = [...supportedFlags].reduce(- (a, x) => a + featureFlagOptions[x].priority,- 0,- );+ const supportScore = [...supportedFlags]+ .filter(isScoredFlag)+ .reduce((a, x) => a + featureFlagOptions[x].priority, 0);const unsupportedFeatures = new Set([...meta.featureFlags]);for (const flag of meta.featureFlags) {@@ -1008,7 +1015,7 @@ export async function buildFallbackList(meta: Meta): Promise<// silently ignoring the user's request and never attempting stealth.// The guard keeps the original list if no stealth-capable engine qualified// (e.g. self-hosted without fire-engine) so scrapes don't break entirely.- if (meta.featureFlags.has("stealthProxy")) {+ if (scoresStealthProxy && meta.featureFlags.has("stealthProxy")) {const stealthCapable = selectedEngines.filter(x => !x.unsupportedFeatures.has("stealthProxy"),);