Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): always route pdf/document downloads through fire-engine when available (#4480)
Gergő Móriczfeb0428e4ff9
diff --git a/apps/api/src/lib/robots-txt.ts b/apps/api/src/lib/robots-txt.tsindex 4e8f5e00f..b1d80b58d 100644--- a/apps/api/src/lib/robots-txt.ts+++ b/apps/api/src/lib/robots-txt.ts@@ -1,18 +1,14 @@import robotsParser, { Robot } from "robots-parser";-import { config } from "../config";import { Logger } from "winston";import { ScrapeOptions, scrapeOptions } from "../controllers/v2/types";import { scrapeURL } from "../scraper/scrapeURL";import { Engine } from "../scraper/scrapeURL/engines";+import { useFireEngine } from "../scraper/scrapeURL/engines/fire-engine/available";import { CostTracking } from "./cost-tracking";import { useIndex } from "../services";const ROBOTS_MAX_AGE = 1 * 24 * 60 * 60 * 1000;-const useFireEngine =- config.FIRE_ENGINE_BETA_URL !== "" &&- config.FIRE_ENGINE_BETA_URL !== undefined;-interface RobotsTxtChecker {robotsTxtUrl: string;robotsTxt: string;diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.tsindex 37f6c71cd..2ad3bcc5f 100644--- a/apps/api/src/scraper/WebScraper/sitemap.ts+++ b/apps/api/src/scraper/WebScraper/sitemap.ts@@ -1,5 +1,4 @@import { parseStringPromise } from "xml2js";-import { config } from "../../config";import { WebCrawler, SITEMAP_LIMIT } from "./crawler";import { scrapeURL } from "../scrapeURL";import { scrapeOptions } from "../../controllers/v2/types";@@ -8,6 +7,7 @@ import { CostTracking } from "../../lib/cost-tracking";import { ScrapeJobTimeoutError } from "../../lib/error";import type { ScrapeOptions } from "../../controllers/v2/types";import { Engine } from "../scrapeURL/engines";+import { useFireEngine } from "../scrapeURL/engines/fire-engine/available";import {ParsedSitemap,parseSitemapXml,@@ -19,10 +19,6 @@ import { promisify } from "node:util";import { fetchFileToBuffer } from "../scrapeURL/engines/utils/downloadFile";import { useIndex } from "../../services";-const useFireEngine =- config.FIRE_ENGINE_BETA_URL !== "" &&- config.FIRE_ENGINE_BETA_URL !== undefined;-const gunzipAsync = promisify(gunzip);export async function getLinksFromSitemap(diff --git a/apps/api/src/scraper/crawler/sitemap.ts b/apps/api/src/scraper/crawler/sitemap.tsindex 040de8599..970b79c2d 100644--- a/apps/api/src/scraper/crawler/sitemap.ts+++ b/apps/api/src/scraper/crawler/sitemap.ts@@ -1,9 +1,9 @@import type { Logger } from "winston";-import { config } from "../../config";import { scrapeOptions, ScrapeOptions } from "../../controllers/v2/types";import { logger as _logger } from "../../lib/logger";import { Engine } from "../scrapeURL/engines";import { scrapeURL } from "../scrapeURL";+import { useFireEngine } from "../scrapeURL/engines/fire-engine/available";import { CostTracking } from "../../lib/cost-tracking";import {processSitemap,@@ -15,10 +15,6 @@ import { promisify } from "node:util";import { SitemapError } from "../../lib/error";import { useIndex } from "../../services";-const useFireEngine =- config.FIRE_ENGINE_BETA_URL !== "" &&- config.FIRE_ENGINE_BETA_URL !== undefined;-type SitemapScrapeOptions = {url: string;maxAge: number;diff --git a/apps/api/src/scraper/scrapeURL/engines/document/index.ts b/apps/api/src/scraper/scrapeURL/engines/document/index.tsindex c6666af19..6cb808691 100644--- a/apps/api/src/scraper/scrapeURL/engines/document/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/document/index.ts@@ -18,8 +18,33 @@ import {documentExtensionFromUrlPath,} from "../../../../lib/document-formats";import { readFile, unlink } from "node:fs/promises";+import { useFireEngine } from "../fire-engine/available";export async function scrapeDocument(meta: Meta): Promise<EngineScrapeResult> {+ // With fire-engine available this engine never downloads files itself:+ // buildFallbackList routes file URLs through the browser engines and the+ // file arrives here via documentPrefetch. Reaching the direct download+ // means either an explicit forceEngine pin (the escape hatch, kept+ // working) or a browser handoff that came back empty+ // (documentPrefetch === null) — signal antibot so the retry loop can give+ // the browser another round trip. In self-hosted deployments (no+ // fire-engine) the direct download stays the primary path. Ordinary+ // pages (no "document" flag) keep declining via EngineUnsuccessfulError+ // so the waterfall just moves on.+ if (+ useFireEngine &&+ meta.internalOptions.forceEngine === undefined &&+ meta.documentPrefetch == null+ ) {+ // A cross-type handoff (a .docx URL serving a PDF) lands in+ // pdfPrefetch: the file is in hand, just not for this engine — decline+ // so the waterfall reaches the engine that can parse it.+ if (meta.pdfPrefetch != null || !meta.featureFlags.has("document")) {+ throw new EngineUnsuccessfulError("document");+ }+ throw new DocumentAntibotError();+ }+let response: Response;let buffer: Buffer;let proxyUsed: "basic" | "stealth" = "basic";diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/available.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/available.tsnew file mode 100644index 000000000..0d8b02e78--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/available.ts@@ -0,0 +1,5 @@+import { config } from "../../../../config";++export const useFireEngine =+ config.FIRE_ENGINE_BETA_URL !== "" &&+ config.FIRE_ENGINE_BETA_URL !== undefined;diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 890712962..662e72685 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -26,6 +26,7 @@ import {isXTwitterUrl,} from "./x-twitter";import { queryEngpickerVerdict, useIndex } from "../../../services";+import { useFireEngine } from "./fire-engine/available";import { hasFormatOfType } from "../../../lib/format-utils";import {getPDFBlocks,@@ -62,9 +63,6 @@ export type Engine =| "wikipedia"| "x-twitter";-const useFireEngine =- config.FIRE_ENGINE_BETA_URL !== "" &&- config.FIRE_ENGINE_BETA_URL !== undefined;const usePlaywright =config.PLAYWRIGHT_MICROSERVICE_URL !== "" &&config.PLAYWRIGHT_MICROSERVICE_URL !== undefined;@@ -740,6 +738,101 @@ export async function buildFallbackList(meta: Meta): Promise<}}+ // File-fetch routing: when fire-engine is available, PDF and document+ // downloads always go through the browser engines, which fetch the file+ // through fire-engine's proxy infrastructure and hand it back to this+ // waterfall via AddFeatureError (specialtyScrapeCheck) — the same route+ // the PDFAntibotError/PDFFetchProxyError and document-equivalent+ // recoveries have always taken, just taken immediately instead of after+ // a wasted direct download. The file engines (pdf, document) then run+ // as pure parsers on the prefetched file, and their own direct undici+ // downloads stay reachable only in self-hosted deployments (no+ // fire-engine) or under an explicit forceEngine pin.+ //+ // This is a bespoke list rather than a flag tweak because the pdf and+ // document flags (priority 100) exist precisely to keep every+ // non-file-capable engine out of the waterfall via the supportScore+ // threshold — the browser engines declare pdf/document: false, so they+ // can never qualify while the flag is set. Clearing the flag instead+ // would let fastMode/atsv requests admit tlsclient, which cannot hand+ // off files at all (the file download handler is chrome-cdp only) and+ // would burn prefetch round trips on null handoffs.+ const fileFetchFlag: FeatureFlag | null = meta.featureFlags.has("document")+ ? "document"+ : meta.featureFlags.has("pdf")+ ? "pdf"+ : null;++ // A handoff of either file type ends the fetch leg: a .pdf URL can+ // legitimately serve a docx (and vice versa), so either prefetch being+ // set means the file is already in hand and the normal waterfall below+ // routes it to the right parser.+ if (+ useFireEngine &&+ fileFetchFlag !== null &&+ meta.pdfPrefetch === undefined &&+ meta.documentPrefetch === undefined &&+ // Lockdown and agentIndexOnly pin forceEngine above, so their+ // index-only semantics win; every other explicit pin is the escape+ // hatch that keeps the file engines' direct downloads working.+ meta.internalOptions.forceEngine === undefined+ ) {+ // Branding needs a rendered HTML page; preserve the historical+ // clean error (this early return would otherwise skip the check at+ // the bottom of this function).+ if (meta.featureFlags.has("branding")) {+ throw new BrandingNotSupportedError(+ fileFetchFlag === "pdf"+ ? "Branding extraction is only supported for HTML web pages. PDFs are not supported."+ : "Branding extraction is only supported for HTML web pages. Documents (docx, xlsx, etc.) are not supported.",+ );+ }++ const browserEngines: Engine[] = meta.featureFlags.has("stealthProxy")+ ? // Explicit stealth: only the stealth variants can honor it. The+ // auto path escalates here on its own via AddFeatureError(["stealthProxy"]).+ [+ "fire-engine;chrome-cdp;stealth",+ "fire-engine(retry);chrome-cdp;stealth",+ ]+ : [+ "fire-engine;chrome-cdp",+ "fire-engine(retry);chrome-cdp",+ "fire-engine;chrome-cdp;stealth",+ "fire-engine(retry);chrome-cdp;stealth",+ ];++ const selectedEngines = [+ // Cache first: an index;documents hit serves the file without any+ // fetch at all. (Plain "index" is the same lookup, but the pdf+ // flag has always threshold-filtered it out for file URLs.)+ ...(shouldUseIndex(meta) ? (["index;documents"] as Engine[]) : []),+ ...browserEngines,+ ].map(engine => {+ const supportedFlags = new Set([+ ...Object.entries(engineOptions[engine].features)+ .filter(+ ([k, v]) => meta.featureFlags.has(k as FeatureFlag) && v === true,+ )+ .map(([k, _]) => k),+ ]);+ const unsupportedFeatures = new Set([...meta.featureFlags]);+ for (const flag of meta.featureFlags) {+ if (supportedFlags.has(flag)) {+ unsupportedFeatures.delete(flag);+ }+ }+ return { engine, unsupportedFeatures };+ });++ meta.logger.info("Selected engines", {+ selectedEngines,+ fileFetchRoute: fileFetchFlag,+ });++ return selectedEngines;+ }+// When fire-engine is available, drop tlsclient and fetch from the general// waterfall: once chrome-cdp (and its retry) fail, degrading to a plain// HTTP client tends to produce bot-walled or otherwise low-quality content,diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex bda10f147..632f60393 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -46,6 +46,7 @@ import {} from "../../../../lib/native-logging";import { withSpan, setSpanAttributes } from "../../../../lib/otel-tracer";import { scrapePDFWithRunPodMU } from "./runpodMU";+import { useFireEngine } from "../fire-engine/available";import { reconcilePageCountWithFirePdf, scrapePDFWithFirePDF } from "./firePDF";import { scrapePDFWithFirePDFAsync } from "./fire-pdf/async";import {@@ -107,6 +108,30 @@ function fetchPdfFileGuardingProxyFailure<T>(}export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {+ // With fire-engine available this engine never downloads files itself:+ // buildFallbackList routes file URLs through the browser engines and the+ // file arrives here via pdfPrefetch. Reaching the direct download means+ // either an explicit forceEngine pin (the escape hatch, kept working) or+ // a browser handoff that came back empty (pdfPrefetch === null) —+ // signal antibot so the retry loop can give the browser another round+ // trip, exactly like a handoff whose bytes failed the PDF sniff. In+ // self-hosted deployments (no fire-engine) the direct download stays the+ // primary path. Ordinary pages (no "pdf" flag) keep declining via+ // EngineUnsuccessfulError so the waterfall just moves on.+ if (+ useFireEngine &&+ meta.internalOptions.forceEngine === undefined &&+ meta.pdfPrefetch == null+ ) {+ // A cross-type handoff (a .pdf URL serving a docx) lands in+ // documentPrefetch: the file is in hand, just not for this engine —+ // decline so the waterfall reaches the engine that can parse it.+ if (meta.documentPrefetch != null || !meta.featureFlags.has("pdf")) {+ throw new EngineUnsuccessfulError("pdf");+ }+ throw new PDFAntibotError();+ }+const shouldParse = shouldParsePDF(meta.options.parsers);const maxPages = getPDFMaxPages(meta.options.parsers);const mode: PDFMode = getPDFMode(meta.options.parsers);@@ -911,7 +936,10 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {},contentType: "application/pdf",- proxyUsed: "basic",+ // Report the proxy that actually delivered the file: a browser+ // handoff may have come through the stealth proxy, while the direct+ // download always uses the basic route.+ proxyUsed: meta.pdfPrefetch?.proxyUsed ?? "basic",};} finally {// Always clean up temp file after we're done with itdiff --git a/apps/api/src/search/fireEngine.ts b/apps/api/src/search/fireEngine.tsindex 8d3f55cf4..0529e499e 100644--- a/apps/api/src/search/fireEngine.ts+++ b/apps/api/src/search/fireEngine.ts@@ -3,10 +3,7 @@ import { SearchResult } from "../../src/lib/entities";import * as Sentry from "@sentry/node";import { logger } from "../lib/logger";import { executeWithRetry, attemptRequest } from "../lib/retry-utils";--const useFireEngine =- config.FIRE_ENGINE_BETA_URL !== "" &&- config.FIRE_ENGINE_BETA_URL !== undefined;+import { useFireEngine } from "../scraper/scrapeURL/engines/fire-engine/available";function hasResults(results: unknown): results is SearchResult[] {return Array.isArray(results) && results.length > 0;diff --git a/apps/api/src/search/v2/fireEngine-v2.ts b/apps/api/src/search/v2/fireEngine-v2.tsindex bbd91817e..88bea6a14 100644--- a/apps/api/src/search/v2/fireEngine-v2.ts+++ b/apps/api/src/search/v2/fireEngine-v2.ts@@ -7,10 +7,7 @@ import {import * as Sentry from "@sentry/node";import { logger } from "../../lib/logger";import { executeWithRetry, attemptRequest } from "../../lib/retry-utils";--const useFireEngine =- config.FIRE_ENGINE_BETA_URL !== "" &&- config.FIRE_ENGINE_BETA_URL !== undefined;+import { useFireEngine } from "../../scraper/scrapeURL/engines/fire-engine/available";function normalizeSearchTypes(type?: SearchResultType | SearchResultType[],