Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): always route pdf/document downloads through fire-engine when available (#4480)

Gergő Móriczfeb0428e4ff9
diff --git a/apps/api/src/lib/robots-txt.ts b/apps/api/src/lib/robots-txt.ts
index 4e8f5e00f..b1d80b58d 100644
--- a/apps/api/src/lib/robots-txt.ts
+++ b/apps/api/src/lib/robots-txt.ts
@@ -1,18 +1,14 @@
import robotsParser, { Robot } from "robots-parser";
-import { config } from "../config";
import { Logger } from "winston";
import { ScrapeOptions, scrapeOptions } from "../controllers/v2/types";
import { scrapeURL } from "../scraper/scrapeURL";
import { Engine } from "../scraper/scrapeURL/engines";
+import { useFireEngine } from "../scraper/scrapeURL/engines/fire-engine/available";
import { CostTracking } from "./cost-tracking";
import { useIndex } from "../services";
const ROBOTS_MAX_AGE = 1 * 24 * 60 * 60 * 1000;
-const useFireEngine =
- config.FIRE_ENGINE_BETA_URL !== "" &&
- config.FIRE_ENGINE_BETA_URL !== undefined;
-
interface RobotsTxtChecker {
robotsTxtUrl: string;
robotsTxt: string;
diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.ts
index 37f6c71cd..2ad3bcc5f 100644
--- a/apps/api/src/scraper/WebScraper/sitemap.ts
+++ b/apps/api/src/scraper/WebScraper/sitemap.ts
@@ -1,5 +1,4 @@
import { parseStringPromise } from "xml2js";
-import { config } from "../../config";
import { WebCrawler, SITEMAP_LIMIT } from "./crawler";
import { scrapeURL } from "../scrapeURL";
import { scrapeOptions } from "../../controllers/v2/types";
@@ -8,6 +7,7 @@ import { CostTracking } from "../../lib/cost-tracking";
import { ScrapeJobTimeoutError } from "../../lib/error";
import type { ScrapeOptions } from "../../controllers/v2/types";
import { Engine } from "../scrapeURL/engines";
+import { useFireEngine } from "../scrapeURL/engines/fire-engine/available";
import {
ParsedSitemap,
parseSitemapXml,
@@ -19,10 +19,6 @@ import { promisify } from "node:util";
import { fetchFileToBuffer } from "../scrapeURL/engines/utils/downloadFile";
import { useIndex } from "../../services";
-const useFireEngine =
- config.FIRE_ENGINE_BETA_URL !== "" &&
- config.FIRE_ENGINE_BETA_URL !== undefined;
-
const gunzipAsync = promisify(gunzip);
export async function getLinksFromSitemap(
diff --git a/apps/api/src/scraper/crawler/sitemap.ts b/apps/api/src/scraper/crawler/sitemap.ts
index 040de8599..970b79c2d 100644
--- a/apps/api/src/scraper/crawler/sitemap.ts
+++ b/apps/api/src/scraper/crawler/sitemap.ts
@@ -1,9 +1,9 @@
import type { Logger } from "winston";
-import { config } from "../../config";
import { scrapeOptions, ScrapeOptions } from "../../controllers/v2/types";
import { logger as _logger } from "../../lib/logger";
import { Engine } from "../scrapeURL/engines";
import { scrapeURL } from "../scrapeURL";
+import { useFireEngine } from "../scrapeURL/engines/fire-engine/available";
import { CostTracking } from "../../lib/cost-tracking";
import {
processSitemap,
@@ -15,10 +15,6 @@ import { promisify } from "node:util";
import { SitemapError } from "../../lib/error";
import { useIndex } from "../../services";
-const useFireEngine =
- config.FIRE_ENGINE_BETA_URL !== "" &&
- config.FIRE_ENGINE_BETA_URL !== undefined;
-
type SitemapScrapeOptions = {
url: string;
maxAge: number;
diff --git a/apps/api/src/scraper/scrapeURL/engines/document/index.ts b/apps/api/src/scraper/scrapeURL/engines/document/index.ts
index c6666af19..6cb808691 100644
--- a/apps/api/src/scraper/scrapeURL/engines/document/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/document/index.ts
@@ -18,8 +18,33 @@ import {
documentExtensionFromUrlPath,
} from "../../../../lib/document-formats";
import { readFile, unlink } from "node:fs/promises";
+import { useFireEngine } from "../fire-engine/available";
export async function scrapeDocument(meta: Meta): Promise<EngineScrapeResult> {
+ // With fire-engine available this engine never downloads files itself:
+ // buildFallbackList routes file URLs through the browser engines and the
+ // file arrives here via documentPrefetch. Reaching the direct download
+ // means either an explicit forceEngine pin (the escape hatch, kept
+ // working) or a browser handoff that came back empty
+ // (documentPrefetch === null) — signal antibot so the retry loop can give
+ // the browser another round trip. In self-hosted deployments (no
+ // fire-engine) the direct download stays the primary path. Ordinary
+ // pages (no "document" flag) keep declining via EngineUnsuccessfulError
+ // so the waterfall just moves on.
+ if (
+ useFireEngine &&
+ meta.internalOptions.forceEngine === undefined &&
+ meta.documentPrefetch == null
+ ) {
+ // A cross-type handoff (a .docx URL serving a PDF) lands in
+ // pdfPrefetch: the file is in hand, just not for this engine — decline
+ // so the waterfall reaches the engine that can parse it.
+ if (meta.pdfPrefetch != null || !meta.featureFlags.has("document")) {
+ throw new EngineUnsuccessfulError("document");
+ }
+ throw new DocumentAntibotError();
+ }
+
let response: Response;
let buffer: Buffer;
let proxyUsed: "basic" | "stealth" = "basic";
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/available.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/available.ts
new file mode 100644
index 000000000..0d8b02e78
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/available.ts
@@ -0,0 +1,5 @@
+import { config } from "../../../../config";
+
+export const useFireEngine =
+ config.FIRE_ENGINE_BETA_URL !== "" &&
+ config.FIRE_ENGINE_BETA_URL !== undefined;
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index 890712962..662e72685 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -26,6 +26,7 @@ import {
isXTwitterUrl,
} from "./x-twitter";
import { queryEngpickerVerdict, useIndex } from "../../../services";
+import { useFireEngine } from "./fire-engine/available";
import { hasFormatOfType } from "../../../lib/format-utils";
import {
getPDFBlocks,
@@ -62,9 +63,6 @@ export type Engine =
| "wikipedia"
| "x-twitter";
-const useFireEngine =
- config.FIRE_ENGINE_BETA_URL !== "" &&
- config.FIRE_ENGINE_BETA_URL !== undefined;
const usePlaywright =
config.PLAYWRIGHT_MICROSERVICE_URL !== "" &&
config.PLAYWRIGHT_MICROSERVICE_URL !== undefined;
@@ -740,6 +738,101 @@ export async function buildFallbackList(meta: Meta): Promise<
}
}
+ // File-fetch routing: when fire-engine is available, PDF and document
+ // downloads always go through the browser engines, which fetch the file
+ // through fire-engine's proxy infrastructure and hand it back to this
+ // waterfall via AddFeatureError (specialtyScrapeCheck) — the same route
+ // the PDFAntibotError/PDFFetchProxyError and document-equivalent
+ // recoveries have always taken, just taken immediately instead of after
+ // a wasted direct download. The file engines (pdf, document) then run
+ // as pure parsers on the prefetched file, and their own direct undici
+ // downloads stay reachable only in self-hosted deployments (no
+ // fire-engine) or under an explicit forceEngine pin.
+ //
+ // This is a bespoke list rather than a flag tweak because the pdf and
+ // document flags (priority 100) exist precisely to keep every
+ // non-file-capable engine out of the waterfall via the supportScore
+ // threshold — the browser engines declare pdf/document: false, so they
+ // can never qualify while the flag is set. Clearing the flag instead
+ // would let fastMode/atsv requests admit tlsclient, which cannot hand
+ // off files at all (the file download handler is chrome-cdp only) and
+ // would burn prefetch round trips on null handoffs.
+ const fileFetchFlag: FeatureFlag | null = meta.featureFlags.has("document")
+ ? "document"
+ : meta.featureFlags.has("pdf")
+ ? "pdf"
+ : null;
+
+ // A handoff of either file type ends the fetch leg: a .pdf URL can
+ // legitimately serve a docx (and vice versa), so either prefetch being
+ // set means the file is already in hand and the normal waterfall below
+ // routes it to the right parser.
+ if (
+ useFireEngine &&
+ fileFetchFlag !== null &&
+ meta.pdfPrefetch === undefined &&
+ meta.documentPrefetch === undefined &&
+ // Lockdown and agentIndexOnly pin forceEngine above, so their
+ // index-only semantics win; every other explicit pin is the escape
+ // hatch that keeps the file engines' direct downloads working.
+ meta.internalOptions.forceEngine === undefined
+ ) {
+ // Branding needs a rendered HTML page; preserve the historical
+ // clean error (this early return would otherwise skip the check at
+ // the bottom of this function).
+ if (meta.featureFlags.has("branding")) {
+ throw new BrandingNotSupportedError(
+ fileFetchFlag === "pdf"
+ ? "Branding extraction is only supported for HTML web pages. PDFs are not supported."
+ : "Branding extraction is only supported for HTML web pages. Documents (docx, xlsx, etc.) are not supported.",
+ );
+ }
+
+ const browserEngines: Engine[] = meta.featureFlags.has("stealthProxy")
+ ? // Explicit stealth: only the stealth variants can honor it. The
+ // auto path escalates here on its own via AddFeatureError(["stealthProxy"]).
+ [
+ "fire-engine;chrome-cdp;stealth",
+ "fire-engine(retry);chrome-cdp;stealth",
+ ]
+ : [
+ "fire-engine;chrome-cdp",
+ "fire-engine(retry);chrome-cdp",
+ "fire-engine;chrome-cdp;stealth",
+ "fire-engine(retry);chrome-cdp;stealth",
+ ];
+
+ const selectedEngines = [
+ // Cache first: an index;documents hit serves the file without any
+ // fetch at all. (Plain "index" is the same lookup, but the pdf
+ // flag has always threshold-filtered it out for file URLs.)
+ ...(shouldUseIndex(meta) ? (["index;documents"] as Engine[]) : []),
+ ...browserEngines,
+ ].map(engine => {
+ const supportedFlags = new Set([
+ ...Object.entries(engineOptions[engine].features)
+ .filter(
+ ([k, v]) => meta.featureFlags.has(k as FeatureFlag) && v === true,
+ )
+ .map(([k, _]) => k),
+ ]);
+ const unsupportedFeatures = new Set([...meta.featureFlags]);
+ for (const flag of meta.featureFlags) {
+ if (supportedFlags.has(flag)) {
+ unsupportedFeatures.delete(flag);
+ }
+ }
+ return { engine, unsupportedFeatures };
+ });
+
+ meta.logger.info("Selected engines", {
+ selectedEngines,
+ fileFetchRoute: fileFetchFlag,
+ });
+
+ return selectedEngines;
+ }
+
// When fire-engine is available, drop tlsclient and fetch from the general
// waterfall: once chrome-cdp (and its retry) fail, degrading to a plain
// HTTP client tends to produce bot-walled or otherwise low-quality content,
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index bda10f147..632f60393 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -46,6 +46,7 @@ import {
} from "../../../../lib/native-logging";
import { withSpan, setSpanAttributes } from "../../../../lib/otel-tracer";
import { scrapePDFWithRunPodMU } from "./runpodMU";
+import { useFireEngine } from "../fire-engine/available";
import { reconcilePageCountWithFirePdf, scrapePDFWithFirePDF } from "./firePDF";
import { scrapePDFWithFirePDFAsync } from "./fire-pdf/async";
import {
@@ -107,6 +108,30 @@ function fetchPdfFileGuardingProxyFailure<T>(
}
export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
+ // With fire-engine available this engine never downloads files itself:
+ // buildFallbackList routes file URLs through the browser engines and the
+ // file arrives here via pdfPrefetch. Reaching the direct download means
+ // either an explicit forceEngine pin (the escape hatch, kept working) or
+ // a browser handoff that came back empty (pdfPrefetch === null) —
+ // signal antibot so the retry loop can give the browser another round
+ // trip, exactly like a handoff whose bytes failed the PDF sniff. In
+ // self-hosted deployments (no fire-engine) the direct download stays the
+ // primary path. Ordinary pages (no "pdf" flag) keep declining via
+ // EngineUnsuccessfulError so the waterfall just moves on.
+ if (
+ useFireEngine &&
+ meta.internalOptions.forceEngine === undefined &&
+ meta.pdfPrefetch == null
+ ) {
+ // A cross-type handoff (a .pdf URL serving a docx) lands in
+ // documentPrefetch: the file is in hand, just not for this engine —
+ // decline so the waterfall reaches the engine that can parse it.
+ if (meta.documentPrefetch != null || !meta.featureFlags.has("pdf")) {
+ throw new EngineUnsuccessfulError("pdf");
+ }
+ throw new PDFAntibotError();
+ }
+
const shouldParse = shouldParsePDF(meta.options.parsers);
const maxPages = getPDFMaxPages(meta.options.parsers);
const mode: PDFMode = getPDFMode(meta.options.parsers);
@@ -911,7 +936,10 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
},
contentType: "application/pdf",
- proxyUsed: "basic",
+ // Report the proxy that actually delivered the file: a browser
+ // handoff may have come through the stealth proxy, while the direct
+ // download always uses the basic route.
+ proxyUsed: meta.pdfPrefetch?.proxyUsed ?? "basic",
};
} finally {
// Always clean up temp file after we're done with it
diff --git a/apps/api/src/search/fireEngine.ts b/apps/api/src/search/fireEngine.ts
index 8d3f55cf4..0529e499e 100644
--- a/apps/api/src/search/fireEngine.ts
+++ b/apps/api/src/search/fireEngine.ts
@@ -3,10 +3,7 @@ import { SearchResult } from "../../src/lib/entities";
import * as Sentry from "@sentry/node";
import { logger } from "../lib/logger";
import { executeWithRetry, attemptRequest } from "../lib/retry-utils";
-
-const useFireEngine =
- config.FIRE_ENGINE_BETA_URL !== "" &&
- config.FIRE_ENGINE_BETA_URL !== undefined;
+import { useFireEngine } from "../scraper/scrapeURL/engines/fire-engine/available";
function hasResults(results: unknown): results is SearchResult[] {
return Array.isArray(results) && results.length > 0;
diff --git a/apps/api/src/search/v2/fireEngine-v2.ts b/apps/api/src/search/v2/fireEngine-v2.ts
index bbd91817e..88bea6a14 100644
--- a/apps/api/src/search/v2/fireEngine-v2.ts
+++ b/apps/api/src/search/v2/fireEngine-v2.ts
@@ -7,10 +7,7 @@ import {
import * as Sentry from "@sentry/node";
import { logger } from "../../lib/logger";
import { executeWithRetry, attemptRequest } from "../../lib/retry-utils";
-
-const useFireEngine =
- config.FIRE_ENGINE_BETA_URL !== "" &&
- config.FIRE_ENGINE_BETA_URL !== undefined;
+import { useFireEngine } from "../../scraper/scrapeURL/engines/fire-engine/available";
function normalizeSearchTypes(
type?: SearchResultType | SearchResultType[],