Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): keep page engines when stealth proxy is unavailable (#5713)

Abimael Martellb3868a988173
diff --git a/apps/api/src/__tests__/snips/v1/deprecation.test.ts b/apps/api/src/__tests__/snips/v1/deprecation.test.ts
index e3728f5dd..3a21b405c 100644
--- a/apps/api/src/__tests__/snips/v1/deprecation.test.ts
+++ b/apps/api/src/__tests__/snips/v1/deprecation.test.ts
@@ -1,6 +1,6 @@
import { describe, it, expect, beforeAll } from "vitest";
import request from "supertest";
-import { itIf, TEST_API_URL } from "../lib";
+import { itIf, scrapeTimeout, TEST_API_URL } from "../lib";
import { sunsetOf } from "../../../lib/deprecations";
import {
DEEP_RESEARCH_LIVE,
@@ -140,7 +140,7 @@ describe("Deprecation warnings on legacy endpoints", () => {
).toBe(true);
expect(res.body.replacement).toBe("/v2/scrape");
},
- 60000,
+ scrapeTimeout,
);
itV1(
diff --git a/apps/api/src/__tests__/snips/v2/stealth-routing-fire-engine.test.ts b/apps/api/src/__tests__/snips/v2/stealth-routing-fire-engine.test.ts
new file mode 100644
index 000000000..90a8ce6e9
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v2/stealth-routing-fire-engine.test.ts
@@ -0,0 +1,116 @@
+/**
+ * Unit test for stealth proxy engine routing via buildFallbackList with
+ * fire-engine configured. These are the fallback lists fire-engine
+ * deployments use, pinned so the self-hosted stealthProxy handling cannot
+ * change them.
+ */
+
+describe("Stealth proxy routing with fire-engine (buildFallbackList)", () => {
+ let buildFallbackList: typeof import("../../../scraper/scrapeURL/engines/index.js").buildFallbackList;
+
+ const overrides: Record<string, string | undefined> = {
+ FIRE_ENGINE_BETA_URL: "http://test-fire-engine",
+ FIRE_EXCHANGE_URL: undefined,
+ INDEX_DATABASE_URL: undefined,
+ PLAYWRIGHT_MICROSERVICE_URL: undefined,
+ };
+ const original: Record<string, string | undefined> = {};
+
+ const setEnv = (env: Record<string, string | undefined>) => {
+ for (const [key, value] of Object.entries(env)) {
+ if (value === undefined) {
+ delete process.env[key];
+ } else {
+ process.env[key] = value;
+ }
+ }
+ };
+
+ beforeAll(async () => {
+ for (const key of Object.keys(overrides)) {
+ original[key] = process.env[key];
+ }
+ setEnv(overrides);
+
+ // Re-import engines fresh so it reads the env vars set above at eval time.
+ vi.resetModules();
+ ({ buildFallbackList } = await import(
+ "../../../scraper/scrapeURL/engines/index.js"
+ ));
+ });
+
+ afterAll(() => {
+ setEnv(original);
+ });
+
+ const engines = async (
+ url: string,
+ featureFlags: string[],
+ proxy: "auto" | "stealth",
+ ) =>
+ (
+ await buildFallbackList({
+ id: "test",
+ url,
+ options: { formats: [], maxAge: 0, proxy },
+ internalOptions: { teamId: "test" },
+ featureFlags: new Set(featureFlags),
+ mock: null,
+ logger: {
+ info: vi.fn(),
+ warn: vi.fn(),
+ debug: vi.fn(),
+ error: vi.fn(),
+ child: vi.fn().mockReturnThis(),
+ },
+ } as any)
+ ).map(f => f.engine);
+
+ const cdp = ["fire-engine;chrome-cdp", "fire-engine(retry);chrome-cdp"];
+ const cdpStealth = [
+ "fire-engine;chrome-cdp;stealth",
+ "fire-engine(retry);chrome-cdp;stealth",
+ ];
+
+ it("routes a regular HTML scrape to chrome-cdp", async () => {
+ expect(await engines("https://example.com/", [], "auto")).toEqual(cdp);
+ });
+
+ it("narrows an HTML scrape that escalated to stealthProxy to the stealth engines", async () => {
+ expect(
+ await engines("https://example.com/", ["stealthProxy"], "auto"),
+ ).toEqual([...cdpStealth, "pdf", "document"]);
+ });
+
+ it("narrows an explicit stealth HTML scrape to the stealth engines", async () => {
+ expect(
+ await engines("https://example.com/", ["stealthProxy"], "stealth"),
+ ).toEqual([...cdpStealth, "pdf", "document"]);
+ });
+
+ it("routes a PDF through the browser engines", async () => {
+ expect(await engines("https://example.com/a.pdf", ["pdf"], "auto")).toEqual(
+ [...cdp, ...cdpStealth],
+ );
+ });
+
+ it("routes a PDF that escalated to stealthProxy through the stealth engines", async () => {
+ expect(
+ await engines(
+ "https://example.com/a.pdf",
+ ["pdf", "stealthProxy"],
+ "auto",
+ ),
+ ).toEqual(cdpStealth);
+ });
+
+ it("routes an explicit stealth document through the stealth engines", async () => {
+ expect(
+ await engines(
+ "https://example.com/a.docx",
+ ["document", "stealthProxy"],
+ "stealth",
+ ),
+ ).toEqual(cdpStealth);
+ });
+});
diff --git a/apps/api/src/__tests__/snips/v2/stealth-routing.test.ts b/apps/api/src/__tests__/snips/v2/stealth-routing.test.ts
new file mode 100644
index 000000000..d69d9bd49
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v2/stealth-routing.test.ts
@@ -0,0 +1,82 @@
+/**
+ * Unit test for stealth proxy engine routing via buildFallbackList.
+ * Without fire-engine no engine fetches a page through a stealth proxy, so an
+ * auto-proxy scrape that escalates to stealthProxy must keep the regular page
+ * engines instead of narrowing to the file parsers.
+ */
+
+describe("Stealth proxy routing without fire-engine (buildFallbackList)", () => {
+ let buildFallbackList: typeof import("../../../scraper/scrapeURL/engines/index.js").buildFallbackList;
+
+ const overrides: Record<string, string | undefined> = {
+ FIRE_ENGINE_BETA_URL: undefined,
+ FIRE_EXCHANGE_URL: undefined,
+ INDEX_DATABASE_URL: undefined,
+ PLAYWRIGHT_MICROSERVICE_URL: "http://test-playwright",
+ };
+ const original: Record<string, string | undefined> = {};
+
+ const setEnv = (env: Record<string, string | undefined>) => {
+ for (const [key, value] of Object.entries(env)) {
+ if (value === undefined) {
+ delete process.env[key];
+ } else {
+ process.env[key] = value;
+ }
+ }
+ };
+
+ beforeAll(async () => {
+ for (const key of Object.keys(overrides)) {
+ original[key] = process.env[key];
+ }
+ setEnv(overrides);
+
+ // Re-import engines fresh so it reads the env vars set above at eval time.
+ vi.resetModules();
+ ({ buildFallbackList } = await import(
+ "../../../scraper/scrapeURL/engines/index.js"
+ ));
+ });
+
+ afterAll(() => {
+ setEnv(original);
+ });
+
+ const buildStubMeta = (featureFlags: string[]) =>
+ ({
+ id: "test",
+ url: "https://example.com/",
+ options: {
+ formats: [],
+ maxAge: 0,
+ proxy: "auto",
+ },
+ internalOptions: { teamId: "test" },
+ featureFlags: new Set(featureFlags),
+ mock: null,
+ logger: {
+ info: vi.fn(),
+ warn: vi.fn(),
+ debug: vi.fn(),
+ error: vi.fn(),
+ child: vi.fn().mockReturnThis(),
+ },
+ }) as any;
+
+ it("keeps the page engines when stealthProxy is added to an HTML scrape", async () => {
+ const before = await buildFallbackList(buildStubMeta([]));
+ const after = await buildFallbackList(buildStubMeta(["stealthProxy"]));
+
+ expect(before.map(f => f.engine)).toEqual(["playwright", "fetch"]);
+ expect(after.map(f => f.engine)).toEqual(["playwright", "fetch"]);
+ });
+
+ it("still routes PDFs to the pdf engine when stealthProxy is set", async () => {
+ const fallback = await buildFallbackList(
+ buildStubMeta(["pdf", "stealthProxy"]),
+ );
+
+ expect(fallback.map(f => f.engine)).toEqual(["pdf"]);
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index 951d979f8..0829cbf12 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -942,10 +942,18 @@ export async function buildFallbackList(meta: Meta): Promise<
}
}
- const prioritySum = [...meta.featureFlags].reduce(
- (a, x) => a + featureFlagOptions[x].priority,
- 0,
- );
+ // Without fire-engine (or its test mocks), no engine fetches a page through
+ // a stealth proxy: only index lookups and the file parsers declare
+ // stealthProxy support. Counting the flag would leave only the file parsers
+ // above the threshold and in the stealth-only narrowing below, so an HTML
+ // page that escalated to stealthProxy would fail on every engine.
+ const scoresStealthProxy = useFireEngine || meta.mock !== null;
+ const isScoredFlag = (flag: FeatureFlag) =>
+ scoresStealthProxy || flag !== "stealthProxy";
+
+ const prioritySum = [...meta.featureFlags]
+ .filter(isScoredFlag)
+ .reduce((a, x) => a + featureFlagOptions[x].priority, 0);
const priorityThreshold = Math.floor(prioritySum / 2);
let selectedEngines: {
engine: Engine;
@@ -984,10 +992,9 @@ export async function buildFallbackList(meta: Meta): Promise<
)
.map(([k, _]) => k),
]);
- const supportScore = [...supportedFlags].reduce(
- (a, x) => a + featureFlagOptions[x].priority,
- 0,
- );
+ const supportScore = [...supportedFlags]
+ .filter(isScoredFlag)
+ .reduce((a, x) => a + featureFlagOptions[x].priority, 0);
const unsupportedFeatures = new Set([...meta.featureFlags]);
for (const flag of meta.featureFlags) {
@@ -1008,7 +1015,7 @@ export async function buildFallbackList(meta: Meta): Promise<
// silently ignoring the user's request and never attempting stealth.
// The guard keeps the original list if no stealth-capable engine qualified
// (e.g. self-hosted without fire-engine) so scrapes don't break entirely.
- if (meta.featureFlags.has("stealthProxy")) {
+ if (scoresStealthProxy && meta.featureFlags.has("stealthProxy")) {
const stealthCapable = selectedEngines.filter(
x => !x.unsupportedFeatures.has("stealthProxy"),
);