Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(pdf): submit large PDFs to FirePDF by GCS reference (#4409)
Abimael Martelld26ad4bbf2fe
diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 3565d93b7..33df37ea6 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -342,6 +342,23 @@ const configSchema = z.object({FIRE_PDF_ASYNC_FORCE_TEAM_IDS: z.string().optional(),FIRE_PDF_ASYNC_DISABLE_TEAM_IDS: z.string().optional(),FIRE_PDF_ASYNC_ALLOW_REQUEST_OVERRIDE: z.stringbool().default(false),+ // Large-PDF by-reference submits (30-256MB files uploaded to GCS and+ // handed to fire-pdf via `input_gcs_uri`). This is an explicit on/off+ // switch, not a percentage: no alternative engine exists at this size,+ // so there is no cohort to sample "out" β only text-only degradation.+ // FIRE_PDF_ENABLE remains the master switch for both paths.+ FIRE_PDF_BY_REFERENCE_ENABLE: z.stringbool().default(true),+ // Bucket that receives large-PDF inputs for by-reference async submits+ // (fire-pdf reads them back via `input_gcs_uri`). fire-pdf only accepts+ // URIs inside its own configured bucket + `inputs/` prefix, so this must+ // match fire-pdf's FIRE_PDF_GCS_BUCKET. Upload failures (e.g. missing+ // IAM grant) fall back to the pre-by-reference behavior for oversized+ // files rather than failing the scrape.+ FIRE_PDF_GCS_INPUT_BUCKET: z+ .string()+ .trim()+ .min(1)+ .default("firecrawl-pdf-pipeline"),// RunPodRUNPOD_MU_API_KEY: z.string().optional(),diff --git a/apps/api/src/lib/gcs-pdf-cache.ts b/apps/api/src/lib/gcs-pdf-cache.tsindex 6cb865109..4f2465af5 100644--- a/apps/api/src/lib/gcs-pdf-cache.ts+++ b/apps/api/src/lib/gcs-pdf-cache.ts@@ -30,8 +30,19 @@ export function createPdfCacheKey(pdfContent: string | Buffer): string {return crypto.createHash("sha256").update(pdfContent).digest("hex");}+/** Cache addressing: historically the key is sha256 of the inline base64+ * payload. Callers that never materialize the base64 (large PDFs submitted+ * by GCS reference) pass a precomputed key instead β namespaced by the+ * caller (e.g. `raw-<sha256-of-bytes>`) so the two keyspaces stay+ * distinct. */+export type PdfCacheKeyInput = string | { key: string };++function resolvePdfCacheKey(input: PdfCacheKeyInput): string {+ return typeof input === "string" ? createPdfCacheKey(input) : input.key;+}+export async function savePdfResultToCache(- pdfContent: string,+ pdfContent: PdfCacheKeyInput,result: CachedPdfResult,provider: PdfCacheProvider = "runpod",variant?: string,@@ -42,7 +53,7 @@ export async function savePdfResultToCache(}const prefix = PROVIDER_PREFIXES[provider];- const cacheKey = createPdfCacheKey(pdfContent);+ const cacheKey = resolvePdfCacheKey(pdfContent);const objectKey = variant ? `${cacheKey}-${variant}` : cacheKey;const bucket = storage.bucket(config.GCS_BUCKET_NAME);const blob = bucket.file(`${prefix}${objectKey}.json`);@@ -89,7 +100,7 @@ export async function savePdfResultToCache(}export async function getPdfResultFromCache(- pdfContent: string,+ pdfContent: PdfCacheKeyInput,provider: PdfCacheProvider = "runpod",variant?: string,): Promise<CachedPdfResult | null> {@@ -99,7 +110,7 @@ export async function getPdfResultFromCache(}const prefix = PROVIDER_PREFIXES[provider];- const cacheKey = createPdfCacheKey(pdfContent);+ const cacheKey = resolvePdfCacheKey(pdfContent);const objectKey = variant ? `${cacheKey}-${variant}` : cacheKey;const bucket = storage.bucket(config.GCS_BUCKET_NAME);const blob = bucket.file(`${prefix}${objectKey}.json`);diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.tsindex 03d351129..72452aa21 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts@@ -11,6 +11,10 @@ import {FirePdfAsyncFailure,scrapePDFWithFirePDFAsync,} from "../fire-pdf/async";+import {+ getPdfResultFromCache,+ savePdfResultToCache,+} from "../../../../../lib/gcs-pdf-cache";import { config } from "../../../../../config";// ββ Fixtures βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ@@ -1136,4 +1140,156 @@ describe("scrapePDFWithFirePDFAsync", () => {expect(delta).toBeGreaterThanOrEqual(5_000 - 100);expect(delta).toBeLessThanOrEqual(30 * 60 * 1_000);});++ describe("by-reference input (large PDFs)", () => {+ const BY_REF = {+ gcsUri: "gs://firecrawl-pdf-pipeline/inputs/scrape-id-test.pdf",+ sha256: "ab".repeat(32),+ sizeBytes: 200 * 1024 * 1024,+ };++ it("submits input_gcs_uri + input_sha256 instead of pdf_b64", async () => {+ vi.mocked(getPdfResultFromCache).mockClear();+ vi.mocked(savePdfResultToCache).mockClear();+ const { fetchImpl, calls } = makeFetchFromSequence([+ {+ matchUrl: /\/jobs$/,+ matchMethod: "POST",+ response: {+ status: 202,+ body: { scrape_id: "scrape-id-test", status: "queued", lane: "xl" },+ },+ },+ {+ matchUrl: /\/jobs\/scrape-id-test$/,+ matchMethod: "GET",+ response: {+ status: 200,+ body: { scrape_id: "x", status: "done", pages_processed: 6543 },+ },+ },+ {+ matchUrl: /\/jobs\/scrape-id-test\/result$/,+ matchMethod: "GET",+ response: {+ status: 200,+ body: {+ schema_version: 1,+ markdown: "# big doc",+ pages_processed: 6543,+ failed_pages: null,+ partial_pages: null,+ },+ },+ },+ ]);++ const result = await scrapePDFWithFirePDFAsync(+ makeMeta(),+ BY_REF,+ undefined,+ 6543,+ undefined,+ { fetchImpl, fallbackImpl: vi.fn(), sleepImpl: noopSleep },+ );++ expect(result.markdown).toBe("# big doc");+ // The by-reference LOOKUP happens at the call site before the input+ // object is uploaded (a hit must skip the transfer, which has already+ // happened by the time this function runs) β so no lookup here, only+ // the save, addressed by the raw-byte sha namespaced apart from+ // inline base64-keyed entries.+ expect(vi.mocked(getPdfResultFromCache)).not.toHaveBeenCalled();+ expect(vi.mocked(savePdfResultToCache)).toHaveBeenCalledWith(+ { key: `raw-${BY_REF.sha256}` },+ expect.anything(),+ "firepdf",+ undefined,+ );+ const body = calls[0].body as Record<string, unknown>;+ expect(body.input_gcs_uri).toBe(BY_REF.gcsUri);+ expect(body.input_sha256).toBe(BY_REF.sha256);+ expect(body.pdf_b64).toBeUndefined();+ expect((body.options as { pages_estimate?: number }).pages_estimate).toBe(+ 6543,+ );+ });++ it("clamps an explicit caller timeout to the 30min ceiling", async () => {+ let submittedBody: any;+ const fetchImpl: any = async (url: string, init: any) => {+ if (/\/jobs$/.test(url) && (init?.method ?? "GET") === "POST") {+ submittedBody = JSON.parse(init.body as string);+ return jsonResp({+ status: 200,+ body: { scrape_id: "x", status: "done", pages_processed: 800 },+ });+ }+ return jsonResp({+ status: 200,+ body: { markdown: "ok", pages_processed: 800 },+ });+ };+ // Long documents need an explicit timeout: the no-budget default+ // stays at 5 minutes because scrapeURLLoop kills no-timeout scrapes+ // at 5 minutes regardless of the advertised FirePDF deadline. A+ // timeout above MAX_DEADLINE_MS must be clamped to it.+ const meta = makeMeta();+ meta.abort.scrapeTimeout = vi.fn(() => 40 * 60 * 1_000);++ await scrapePDFWithFirePDFAsync(+ meta,+ { ...BY_REF },+ undefined,+ 800,+ undefined,+ {+ fetchImpl,+ fallbackImpl: vi.fn(),+ sleepImpl: noopSleep,+ },+ );++ const delta = new Date(submittedBody.deadline_at).getTime() - Date.now();+ expect(delta).toBeGreaterThan(29 * 60 * 1_000);+ expect(delta).toBeLessThanOrEqual(30 * 60 * 1_000);+ });++ it("throws (never falls back) for by-reference under ZDR", async () => {+ const fallback = vi.fn();+ await expect(+ scrapePDFWithFirePDFAsync(+ makeMeta({+ internalOptions: {+ zeroDataRetention: true,+ teamId: "team-x",+ teamConcurrency: 12,+ crawlId: undefined,+ },+ }),+ { ...BY_REF },+ undefined,+ 100,+ undefined,+ { fetchImpl: vi.fn(), fallbackImpl: fallback, sleepImpl: noopSleep },+ ),+ ).rejects.toThrow(/zero data retention/);+ expect(fallback).not.toHaveBeenCalled();+ });++ it("rejects a by-reference submit without a positive pages estimate", async () => {+ for (const pagesEstimate of [undefined, 0, -1]) {+ await expect(+ scrapePDFWithFirePDFAsync(+ makeMeta(),+ { ...BY_REF },+ undefined,+ pagesEstimate,+ undefined,+ { fetchImpl: vi.fn(), fallbackImpl: vi.fn(), sleepImpl: noopSleep },+ ),+ ).rejects.toThrow(/pages estimate/);+ }+ });+ });});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.tsindex 53e135bd6..6e680ec11 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts@@ -10,6 +10,7 @@ import { tryGetCached, maybeSaveResult } from "./cache";import { firePdfAsyncTotalDurationSeconds } from "./metrics";import { pollUntilTerminal } from "./poll";import { fetchResult } from "./result";+import type { FirePdfByReferenceInput } from "./by-reference";import { FIRE_PDF_ASYNC_MIN_REMAINING_MS } from "./routing";import { POLL_FLOOR_MS, POLL_TIMEOUT_BUFFER_MS } from "./schema";import { submitJob, SubmitJobMayHaveBeenAcceptedError } from "./submit";@@ -32,7 +33,11 @@ type FirePdfAsyncDeps = {export async function scrapePDFWithFirePDFAsync(meta: Meta,- base64Content: string,+ /** Inline base64 (string, the historical shape) or a pre-uploaded GCS+ * reference for large files. By-reference has no sync fallback β the+ * bytes don't fit fire-pdf's inline paths β so infra fallbacks below+ * only apply to the string form. */+ input: string | FirePdfByReferenceInput,maxPages?: number,pagesProcessed?: number,mode?: PDFMode,@@ -46,10 +51,18 @@ export async function scrapePDFWithFirePDFAsync(const sleep = deps.sleepImpl ?? defaultSleep;const now = deps.nowImpl ?? Date.now;const random = deps.randomImpl ?? Math.random;+ const base64Content = typeof input === "string" ? input : undefined;// Async persists inputs and queue state, so ZDR is excluded until that// lifecycle has an explicit delete-on-completion contract.if (meta.internalOptions.zeroDataRetention) {+ if (base64Content === undefined) {+ // By-reference already persisted the input object; the routing layer+ // must never send ZDR traffic here.+ throw new Error(+ "fire-pdf by-reference submit is not available under zero data retention",+ );+ }return fallbackImpl(meta,base64Content,@@ -62,16 +75,32 @@ export async function scrapePDFWithFirePDFAsync();}- const cached = await tryGetCached(- meta,- base64Content,- mode,- maxPages,- pagesProcessed,- includePageMarkdown,- includeBlocks,- pageMarkers,- );+ // Cache addressing: inline submits keep the historical key (sha256 of+ // the base64 payload); by-reference submits use a `raw-` prefixed key+ // over the raw-byte sha, so repeat scrapes of the same large document+ // don't reprocess it. The two keyspaces are deliberately distinct β an+ // inline and a by-reference parse of the same document do not share+ // entries. The LOOKUP for by-reference happens at the call site BEFORE+ // the input object is uploaded (a hit must skip the 30-256MB transfer,+ // which has already happened by the time this function runs); only the+ // inline path looks up here. Both paths save here.+ const cacheInput =+ typeof input === "string"+ ? input+ : { key: `raw-${input.sha256.toLowerCase()}` };+ const cached =+ typeof input === "string"+ ? await tryGetCached(+ meta,+ cacheInput,+ mode,+ maxPages,+ pagesProcessed,+ includePageMarkdown,+ includeBlocks,+ pageMarkers,+ )+ : null;if (cached) return cached;meta.abort.throwIfAborted();@@ -88,6 +117,11 @@ export async function scrapePDFWithFirePDFAsync(if (!baseUrl) {// Should be unreachable β call site checks this β but fall back rather// than crash if a route somehow bypasses the gate.+ if (base64Content === undefined) {+ throw new Error(+ "fire-pdf by-reference submit requires FIRE_PDF_BASE_URL",+ );+ }return fallbackImpl(meta,base64Content,@@ -102,6 +136,10 @@ export async function scrapePDFWithFirePDFAsync(const overallStartedAt = now();const submitTime = now();+ // Note for large by-reference documents: the no-budget fallback inside+ // computeDeadlineMs is 5 minutes because scrapeURLLoop kills no-timeout+ // scrapes at 5 minutes anyway. Callers wanting the full multi-minute+ // window for big documents must pass an explicit `timeout`.const deadlineFromNow = computeDeadlineMs(remainingMs);const deadlineAt = new Date(submitTime + deadlineFromNow).toISOString();const pollingDeadline = submitTime + deadlineFromNow + POLL_TIMEOUT_BUFFER_MS;@@ -125,7 +163,14 @@ export async function scrapePDFWithFirePDFAsync(const submit = await submitJob({meta,baseUrl,- base64Content,+ input:+ typeof input === "string"+ ? { kind: "inline", base64Content: input }+ : {+ kind: "byReference",+ gcsUri: input.gcsUri,+ sha256: input.sha256,+ },maxPages,pagesProcessed,mode,@@ -232,7 +277,7 @@ export async function scrapePDFWithFirePDFAsync(await maybeSaveResult({meta,- base64Content,+ base64Content: cacheInput,mode,maxPages,includePageMarkdown,diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.tsnew file mode 100644index 000000000..b6660ba6b--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts@@ -0,0 +1,186 @@+import { createHash } from "node:crypto";+import { createReadStream } from "node:fs";+import { Transform } from "node:stream";+import { pipeline } from "node:stream/promises";+import type { Meta } from "../../..";+import { config } from "../../../../../config";+import { storage } from "../../../../../lib/gcs-jobs";++/** Input handle for a FirePDF async submit that travels by GCS reference+ * instead of inline base64. Produced by {@link uploadPdfInputForFirePdf}. */+export type FirePdfByReferenceInput = {+ gcsUri: string;+ sha256: string;+ sizeBytes: number;+};++/** Uploading 256MB in-cluster to GCS is seconds; this bound exists so a+ * stuck stream cannot hold a scrape slot indefinitely. */+const UPLOAD_TIMEOUT_MS = 120_000;++/**+ * Whether by-reference FirePDF routing is configured and permitted for this+ * request, judged from signals available before the file is downloaded.+ * Both the download-size admission and the routing gate use this ONE+ * predicate; the routing gate then adds the file-dependent conditions+ * (size window, page count, MinerU diversion).+ *+ * ZDR is excluded because the by-reference input object persists in GCS.+ * A forced FirePDF request (pages/blocks/markers) needs only the base URL,+ * mirroring the inline path's rule; otherwise both the master switch and+ * the by-reference switch must be on.+ */+export function byReferenceConfigured(+ meta: Meta,+ forceFirePdfRequested: boolean,+): boolean {+ return (+ !meta.internalOptions.zeroDataRetention &&+ !!config.FIRE_PDF_BASE_URL &&+ (forceFirePdfRequested ||+ (!!config.FIRE_PDF_ENABLE && config.FIRE_PDF_BY_REFERENCE_ENABLE))+ );+}++/**+ * Stream a downloaded PDF from its temp file into the fire-pdf input bucket+ * so the async pipeline can fetch it by reference. Single pass: the sha-256+ * (fire-pdf's idempotency identity) is computed through a transform inside+ * the upload pipeline, so the file is read exactly once and never buffered.+ * A timeout or scrape cancellation aborts the transfer itself (both streams+ * are destroyed) β the caller may unlink the temp file as soon as this+ * returns.+ *+ * Cleanup contract: the uploaded object is deliberately NOT deleted here or+ * on job completion. fire-pdf's committed-retry replay depends on the input+ * object outliving the job, and the bucket's prefix-scoped lifecycle policy+ * (inputs/ deleted after 1 day, owned with the bucket in infra) is the+ * cleanup mechanism.+ *+ * Returns null on any failure (missing bucket grant, timeout, transport):+ * the caller falls back to the pre-by-reference behavior for oversized+ * files instead of failing the scrape on infra misconfiguration.+ */+/** Streaming sha-256 of a file on disk. The by-reference cache identity β+ * computed BEFORE any upload so a cache hit skips the 30-256MB transfer+ * entirely. A disk read is orders of magnitude cheaper than the upload it+ * can save. */+export async function sha256OfFile(+ path: string,+ signal?: AbortSignal,+): Promise<string> {+ const hash = createHash("sha256");+ for await (const chunk of createReadStream(path, { signal })) {+ hash.update(chunk as Buffer);+ }+ return hash.digest("hex");+}++export async function uploadPdfInputForFirePdf(+ meta: Meta,+ tempFilePath: string,+ sizeBytes: number,+ opts?: {+ /** sha-256 already computed over this exact file (e.g. for the+ * pre-upload cache check) β skips the in-pipeline hash pass. */+ precomputedSha256?: string;+ },+): Promise<FirePdfByReferenceInput | null> {+ const bucketName = config.FIRE_PDF_GCS_INPUT_BUCKET;+ // Scrape ids are UUIDv7 (time-ordered); a short hash prefix spreads the+ // keys across GCS partitions, same convention as gcs-jobs.ts.+ const keyPrefix = createHash("sha256")+ .update(meta.id)+ .digest("hex")+ .slice(0, 8);+ const objectKey = `inputs/${keyPrefix}-${meta.id}.pdf`;+ const startedAt = Date.now();+ try {+ // Both the hash pass and the upload stop on scrape cancellation as+ // well as the local timeout β a cancelled request must not keep+ // reading and uploading hundreds of MB it can no longer use.+ const timeoutAbort = new AbortController();+ const signal = AbortSignal.any([+ timeoutAbort.signal,+ meta.abort.asSignal(),+ ]);+ // A first asSignal() call AFTER cancellation returns a signal whose+ // abort listeners were attached post-abort and never fire β check the+ // manager directly before starting the timer or any stream.+ meta.abort.throwIfAborted();+ const timer = setTimeout(+ () =>+ timeoutAbort.abort(+ new Error(`GCS input upload timed out after ${UPLOAD_TIMEOUT_MS}ms`),+ ),+ UPLOAD_TIMEOUT_MS,+ );+ const hash =+ opts?.precomputedSha256 === undefined ? createHash("sha256") : null;+ try {+ // Stream construction lives inside the timer-owning try: a+ // synchronous createWriteStream throw must still clear the timeout.+ const writeStream = storage+ .bucket(bucketName)+ .file(objectKey)+ .createWriteStream({+ resumable: true,+ metadata: {+ contentType: "application/pdf",+ metadata: { scrape_id: meta.id, source: "firecrawl" },+ },+ });+ if (hash) {+ const hashThrough = new Transform({+ transform(chunk, _encoding, callback) {+ hash.update(chunk as Buffer);+ callback(null, chunk);+ },+ });+ await pipeline(+ createReadStream(tempFilePath),+ hashThrough,+ writeStream,+ {+ signal,+ },+ );+ } else {+ await pipeline(createReadStream(tempFilePath), writeStream, { signal });+ }+ } finally {+ clearTimeout(timer);+ }+ const sha256 = opts?.precomputedSha256 ?? hash!.digest("hex");+ meta.logger.info("Uploaded large PDF for by-reference FirePDF submit", {+ method: "scrapePDF/firePdfByReference",+ event: "fire_pdf_by_reference_uploaded",+ scrape_id: meta.id,+ size_bytes: sizeBytes,+ duration_ms: Date.now() - startedAt,+ gcs_uri: `gs://${bucketName}/${objectKey}`,+ });+ return {+ gcsUri: `gs://${bucketName}/${objectKey}`,+ sha256,+ sizeBytes,+ };+ } catch (error) {+ // A cancelled scrape propagates as an abort, not a fallback β the+ // legacy chain must not keep processing a request nobody is waiting on.+ meta.abort.throwIfAborted();+ meta.logger.warn(+ "Large-PDF GCS input upload failed; falling back to legacy handling",+ {+ method: "scrapePDF/firePdfByReference",+ event: "fire_pdf_by_reference_upload_failed",+ scrape_id: meta.id,+ team_id: meta.internalOptions.teamId,+ size_bytes: sizeBytes,+ bucket: bucketName,+ error,+ },+ );+ return null;+ }+}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.tsindex d62fc723e..9c963edad 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts@@ -4,6 +4,7 @@ import type { PDFProcessorResult } from "../types";import {getPdfResultFromCache,savePdfResultToCache,+ type PdfCacheKeyInput,} from "../../../../../lib/gcs-pdf-cache";import { firePdfBlockPagesSchema } from "./schema";@@ -143,7 +144,7 @@ export function cacheKeyShape(export async function tryGetCached(meta: Meta,- base64Content: string,+ base64Content: PdfCacheKeyInput,mode: PDFMode | undefined,maxPages: number | undefined,pagesProcessed: number | undefined,@@ -205,7 +206,7 @@ export async function tryGetCached(export async function maybeSaveResult(args: {meta: Meta;- base64Content: string;+ base64Content: PdfCacheKeyInput;mode: PDFMode | undefined;maxPages: number | undefined;includePageMarkdown: boolean;diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.tsindex 716cc4791..8fa4653a2 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts@@ -12,10 +12,18 @@ type SubmitOutcome = {alreadyDone: boolean;};+/** How the PDF bytes reach fire-pdf: inline base64 for small files, or a+ * GCS reference (pre-uploaded to fire-pdf's input bucket) for large ones.+ * By-reference submits require a positive `pages_estimate` β fire-pdf has+ * no bytes to probe at admission time. */+type FirePdfSubmitInput =+ | { kind: "inline"; base64Content: string }+ | { kind: "byReference"; gcsUri: string; sha256: string };+type SubmitArgs = {meta: Meta;baseUrl: string;- base64Content: string;+ input: FirePdfSubmitInput;maxPages: number | undefined;pagesProcessed: number | undefined;mode: PDFMode | undefined;@@ -57,7 +65,7 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {const {meta,baseUrl,- base64Content,+ input,maxPages,pagesProcessed,mode,@@ -70,8 +78,22 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {} = args;const scrapeId = meta.id;+ if (+ input.kind === "byReference" &&+ (pagesProcessed === undefined || pagesProcessed <= 0)+ ) {+ // fire-pdf rejects by-reference submits without a positive+ // pages_estimate (400 missing_pages_estimate); fail here with the+ // clearer local error instead of a wire round-trip.+ throw new Error(+ "fire-pdf by-reference submit requires a positive pages estimate",+ );+ }+const body = {- pdf_b64: base64Content,+ ...(input.kind === "inline"+ ? { pdf_b64: input.base64Content }+ : { input_gcs_uri: input.gcsUri, input_sha256: input.sha256 }),scrape_id: scrapeId,source: "firecrawl" as const,zdr: false as const,diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.tsindex 594856a1b..0d0f30cb0 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts@@ -39,9 +39,14 @@ export function nextPollDelay(}export function computeDeadlineMs(scrapeTimeoutMs: number | undefined): number {- // 5min default when there's no scrape budget (CLI/tests). Routing rejects- // budgets that are too short; only cap the upper bound here so we never- // advertise more time to FirePDF than the caller actually has.+ // 5min default when there's no scrape budget. This matches the usable+ // window exactly: scrapeURLLoop races every engine against+ // `scrapeTimeout() ?? 300000`, so a no-timeout scrape dies at 5 minutes+ // regardless of what we advertise to FirePDF. Long documents (large+ // by-reference PDFs especially) need an explicit caller `timeout`, which+ // unlocks up to MAX_DEADLINE_MS here and extends the loop's race alike.+ // Only cap the upper bound so we never advertise more time to FirePDF+ // than the caller actually has.const fallback = 5 * 60 * 1_000;const candidate = scrapeTimeoutMs ?? fallback;return Math.min(MAX_DEADLINE_MS, candidate);diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 9c6536dbf..b9d35ed58 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -11,7 +11,7 @@ import {RemoveFeatureError,EngineUnsuccessfulError,} from "../../error";-import { open, readFile, unlink } from "node:fs/promises";+import { open, readFile, stat, unlink } from "node:fs/promises";import type { Response } from "undici";import { AbortManagerThrownError } from "../../lib/abortManager";import {@@ -26,6 +26,8 @@ import {import type { PDFMode } from "../../../../controllers/v2/types";import { processPdf, detectPdf } from "@mendable/firecrawl-rs";import {+ FIRE_PDF_BY_REFERENCE_MAX_FILE_SIZE,+ FIRE_PDF_INLINE_HARD_MAX_FILE_SIZE,FIRE_PDF_MAX_FILE_SIZE,MAX_FILE_SIZE,MILLISECONDS_PER_PAGE,@@ -40,6 +42,12 @@ import { withSpan, setSpanAttributes } from "../../../../lib/otel-tracer";import { scrapePDFWithRunPodMU } from "./runpodMU";import { reconcilePageCountWithFirePdf, scrapePDFWithFirePDF } from "./firePDF";import { scrapePDFWithFirePDFAsync } from "./fire-pdf/async";+import {+ byReferenceConfigured,+ sha256OfFile,+ uploadPdfInputForFirePdf,+} from "./fire-pdf/by-reference";+import { cacheKeyShape, tryGetCached } from "./fire-pdf/cache";import { decideFirePdfAsyncRoute } from "./fire-pdf/routing";import { scrapePDFWithParsePDF } from "./pdfParse";import { toPublicBlocks } from "./blocks";@@ -139,6 +147,37 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {}}+ const forceFirePDF =+ (!!meta.options.__forceFirePDF ||+ includePageMarkdown ||+ includeBlocks ||+ pageMarkers) &&+ !!config.FIRE_PDF_BASE_URL;++ // The MinerU diversion draw happens BEFORE the download so the admission+ // cap can honor it: a diverted request cannot use the by-reference route,+ // so it must keep the historical download cap instead of pulling up to+ // 256MB it would only hand to pdf-parse. Forced Fire PDF takes+ // precedence β don't divert those requests.+ const routeToMinerU =+ !forceFirePDF &&+ config.MINERU_PERCENT > 0 &&+ Math.random() * 100 < config.MINERU_PERCENT;++ // Only admit large downloads when the by-reference FirePDF path is even+ // reachable for this request (same predicate the routing gate uses; the+ // gate adds the signals that need the file first). Otherwise keep the+ // historical cap β an oversized file would only burn bandwidth and temp+ // disk to fall through to text-only extraction.+ // Fast mode is excluded: its hard cost ceiling skips the whole OCR/+ // FirePDF chain (skipOCR), so the by-reference route it would justify is+ // unreachable and the raised admission would only buffer bytes for the+ // native path.+ const byReferenceReachable =+ !routeToMinerU &&+ mode !== "fast" &&+ byReferenceConfigured(meta, forceFirePDF);+const { response, tempFilePath } =meta.pdfPrefetch !== undefined && meta.pdfPrefetch !== null? { response: meta.pdfPrefetch, tempFilePath: meta.pdfPrefetch.filePath }@@ -150,7 +189,11 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {headers: meta.options.headers,signal: meta.abort.asSignal(),},- PDF_DOWNLOAD_MAX_FILE_SIZE,+ // Parse path streams to disk and can hand large files to FirePDF+ // by GCS reference, so it admits more than the raw fetch path.+ byReferenceReachable+ ? FIRE_PDF_BY_REFERENCE_MAX_FILE_SIZE+ : PDF_DOWNLOAD_MAX_FILE_SIZE,);try {@@ -194,22 +237,9 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {let shadowIneligibleReason: string | null | undefined;let shadowPagesNeedingOcr: number[] | undefined;- const forceFirePDF =- (!!meta.options.__forceFirePDF ||- includePageMarkdown ||- includeBlocks ||- pageMarkers) &&- !!config.FIRE_PDF_BASE_URL;const rustEnabled = !!config.PDF_RUST_EXTRACT_ENABLE;const logger = meta.logger.child({ method: "scrapePDF/processPdf" });- // Route a percentage of traffic directly to MinerU, bypassing Rust extraction.- // Forced Fire PDF takes precedence β don't divert those requests.- const routeToMinerU =- !forceFirePDF &&- config.MINERU_PERCENT > 0 &&- Math.random() * 100 < config.MINERU_PERCENT;-if (routeToMinerU) {logger.info("Routing to MinerU via MINERU_PERCENT", {mineruPercent: config.MINERU_PERCENT,@@ -405,10 +435,199 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {// unless we explicitly routed to MinerU via MINERU_PERCENT.const skipOCR =rustEnabled && mode === "fast" && !routeToMinerU && !forceFirePDF;++ // Large PDFs can't travel inline as base64 JSON (fire-pdf's body limit,+ // V8 string ceilings, worker memory), so they go to the fire-pdf async+ // pipeline by GCS reference β streamed from the temp file, never+ // buffered. ZDR stays out: the by-reference input object persists in+ // GCS. MinerU-diverted traffic keeps its route decision (MinerU can't+ // take these sizes, so the legacy chain below just skips through).+ // By-reference has its own explicit switch instead of riding+ // FIRE_PDF_PERCENT: there is no alternative engine at this size, so a+ // sampled-out cohort would only degrade to text-only extraction. The+ // shared predicate matches the download-admission decision above; only+ // the file-dependent conditions are added here.if (!result && !skipOCR) {- const pdfBuffer = await readFile(tempFilePath);- const fileSizeBytes = pdfBuffer.length;- const base64Content = pdfBuffer.toString("base64");+ const fileSizeBytes = (await stat(tempFilePath)).size;+ // mode !== "fast" mirrors the admission gate above: fast mode keeps+ // its pre-by-reference behavior on every path (with Rust extraction+ // disabled, skipOCR alone would not exclude it here).+ const useFirePdfByReference =+ !routeToMinerU &&+ mode !== "fast" &&+ byReferenceConfigured(meta, forceFirePDF) &&+ fileSizeBytes >= FIRE_PDF_MAX_FILE_SIZE &&+ fileSizeBytes <= FIRE_PDF_BY_REFERENCE_MAX_FILE_SIZE;++ if (useFirePdfByReference) {+ if (effectivePageCount <= 0) {+ // fire-pdf can't probe pages without the bytes, so by-reference+ // submits require our page count. Fall through to the legacy+ // chain (status quo for oversized files).+ meta.logger.warn(+ "Large PDF has no page-count estimate; cannot submit by reference",+ {+ method: "scrapePDF",+ event: "fire_pdf_by_reference_no_pages",+ file_size_bytes: fileSizeBytes,+ scrape_id: meta.id,+ team_id: meta.internalOptions.teamId,+ },+ );+ } else {+ // Cache BEFORE upload: the raw-byte sha is the by-reference cache+ // identity, and it must be checked before the 30-256MB transfer β+ // a repeat scrape of the same document should cost one streamed+ // disk read, not a full re-upload. scrapePDFWithFirePDFAsync+ // deliberately skips the by-reference lookup for the same reason+ // (it runs post-upload) and only saves. Uncacheable requests+ // (fast mode / maxPages) skip the pre-hash entirely β it could+ // never produce a hit, and the upload computes its own hash+ // in-pipeline. A failed pre-hash falls through to the upload+ // path (legacy fallback semantics), never errors the scrape.+ const { cacheable: byRefCacheable } = cacheKeyShape(+ mode,+ maxPages,+ includePageMarkdown,+ includeBlocks,+ pageMarkers,+ );+ let localSha256: string | undefined;+ if (byRefCacheable) {+ try {+ localSha256 = await sha256OfFile(+ tempFilePath,+ meta.abort.asSignal(),+ );+ } catch (error) {+ meta.logger.warn(+ "Pre-upload hash of large PDF failed; continuing without cache lookup",+ {+ method: "scrapePDF/firePdfByReference",+ error,+ scrape_id: meta.id,+ },+ );+ }+ }+ const cachedByRef = localSha256+ ? await tryGetCached(+ meta,+ { key: `raw-${localSha256}` },+ mode,+ maxPages,+ effectivePageCount,+ includePageMarkdown,+ includeBlocks,+ pageMarkers,+ )+ : null;+ // A scrape cancelled during the hash/lookup must not return a+ // success out of the cache.+ meta.abort.throwIfAborted();+ if (cachedByRef) {+ result = cachedByRef;+ effectivePageCount = reconcilePageCountWithFirePdf(+ effectivePageCount,+ result,+ );+ }+ const uploaded = result+ ? null+ : await uploadPdfInputForFirePdf(+ meta,+ tempFilePath,+ fileSizeBytes,+ {+ precomputedSha256: localSha256,+ },+ );+ if (uploaded) {+ try {+ result = await scrapePDFWithFirePDFAsync(+ {+ ...meta,+ logger: meta.logger.child({+ method: "scrapePDF/firePDFAsyncByReference",+ }),+ },+ uploaded,+ maxPages,+ effectivePageCount,+ mode,+ undefined,+ includePageMarkdown,+ includeBlocks,+ pageMarkers,+ );+ effectivePageCount = reconcilePageCountWithFirePdf(+ effectivePageCount,+ result,+ );+ } catch (error) {+ // No inline retry exists at this size, and the legacy chain+ // below would silently degrade a large document to text-only+ // extraction. Surface the failure instead.+ meta.logger.error(+ "FirePDF by-reference scrape failed (no fallback at this size)",+ {+ method: "scrapePDF/firePDFAsyncByReference",+ error,+ file_size_bytes: fileSizeBytes,+ scrape_id: meta.id,+ team_id: meta.internalOptions.teamId,+ },+ );+ throw error;+ }+ }+ // Upload failure: fall through to the legacy chain β the+ // oversized-skip warning below still fires, preserving the+ // pre-by-reference behavior.+ }+ }+ }++ if (!result && !skipOCR) {+ const fileSizeBytes = (await stat(tempFilePath)).size;+ // Only materialize the base64 payload for engines that can accept it+ // inline: FirePDF (<30MB, or forced up to fire-pdf's wire ceiling)+ // and MinerU (<19MB). Files above that reach this chain only as+ // fallthrough (ZDR, MinerU-diverted, by-reference failure) and go+ // straight to pdf-parse, which reads from disk β buffering and+ // base64-encoding hundreds of MB here would only burn worker memory.+ // Inline bytes are only materialized when an engine that can accept+ // them on THIS route and size actually exists: FirePDF inline (not+ // MinerU-diverted; under its cap, or forced up to the wire ceiling)+ // or RunPod MU (under its own cap). Everything else goes straight to+ // disk-based pdf-parse without a wasted base64 pass.+ const firePdfInlineUsable =+ (forceFirePDF ||+ (!routeToMinerU &&+ !!config.FIRE_PDF_ENABLE &&+ !!config.FIRE_PDF_BASE_URL)) &&+ (fileSizeBytes < FIRE_PDF_MAX_FILE_SIZE ||+ (forceFirePDF &&+ fileSizeBytes <= FIRE_PDF_INLINE_HARD_MAX_FILE_SIZE));+ const runpodMuUsable =+ !forceFirePDF &&+ fileSizeBytes < MAX_FILE_SIZE &&+ !!config.RUNPOD_MU_API_KEY &&+ !!config.RUNPOD_MU_POD_ID;+ const inlineEligible = firePdfInlineUsable || runpodMuUsable;+ const base64Content = inlineEligible+ ? (await readFile(tempFilePath)).toString("base64")+ : undefined;++ if (!result && forceFirePDF && base64Content === undefined) {+ // Forced FirePDF (pages/blocks/markers) with no viable transport:+ // the file exceeds the inline ceiling and the by-reference path was+ // unavailable (ZDR) or failed. Erroring beats returning an empty+ // document as a 200.+ throw new Error(+ `PDF (${fileSizeBytes} bytes) exceeds the FirePDF inline ceiling and by-reference submission was unavailable`,+ );+ }if (!forceFirePDF &&@@ -432,12 +651,13 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {// forceFirePDF always wins; skip percentage-based Fire PDF when// we explicitly routed to MinerU via MINERU_PERCENT.const useFirePDF =- forceFirePDF ||- (!routeToMinerU &&- config.FIRE_PDF_ENABLE &&- config.FIRE_PDF_BASE_URL &&- fileSizeBytes < FIRE_PDF_MAX_FILE_SIZE &&- Math.random() * 100 < config.FIRE_PDF_PERCENT);+ base64Content !== undefined &&+ (forceFirePDF ||+ (!routeToMinerU &&+ config.FIRE_PDF_ENABLE &&+ config.FIRE_PDF_BASE_URL &&+ fileSizeBytes < FIRE_PDF_MAX_FILE_SIZE &&+ Math.random() * 100 < config.FIRE_PDF_PERCENT));if (useFirePDF) {// Async is a server-controlled cohort within traffic already selected@@ -596,6 +816,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {!result &&!forceFirePDF &&fileSizeBytes < MAX_FILE_SIZE &&+ base64Content !== undefined &&config.RUNPOD_MU_API_KEY &&config.RUNPOD_MU_POD_ID) {diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/types.tsindex c021e134f..e886d5b69 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts@@ -64,6 +64,21 @@ export type PdfMetadata = {};export const MAX_FILE_SIZE = 19 * 1024 * 1024; // 19MB+/** Above this, FirePDF submits go by GCS reference (async /jobs with+ * `input_gcs_uri`) instead of inline base64 β inline bodies are capped by+ * fire-pdf's JSON body limit and V8 string ceilings. */export const FIRE_PDF_MAX_FILE_SIZE = 30 * 1024 * 1024; // 30MB+/** Absolute ceiling for inline (base64 JSON) FirePDF submits, forced or+ * not: fire-pdf's 100MB body limit divided by base64 inflation is ~74MB+ * raw; 70MB leaves margin for the JSON envelope. Beyond this the inline+ * path is a guaranteed 413, so callers shouldn't even build the string. */+export const FIRE_PDF_INLINE_HARD_MAX_FILE_SIZE = 70 * 1024 * 1024; // 70MB+/** Ceiling for by-reference FirePDF submits, and therefore for parse-path+ * downloads (which stream to disk and hand large files to FirePDF by GCS+ * reference). Mirrors fire-pdf's FIRE_PDF_GCS_INPUT_MAX_BYTES default β+ * raise the two together. */+export const FIRE_PDF_BY_REFERENCE_MAX_FILE_SIZE = 256 * 1024 * 1024; // 256MB+/** Raw (non-parsed) PDF fetches return the file base64'd inline in the+ * response, so they keep a tighter cap than the parse path. */export const PDF_DOWNLOAD_MAX_FILE_SIZE = 50 * 1024 * 1024; // 50MBexport const MILLISECONDS_PER_PAGE = 150;