Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

feat(pdf): submit large PDFs to FirePDF by GCS reference (#4409)

Abimael Martelld26ad4bbf2fe
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 3565d93b7..33df37ea6 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -342,6 +342,23 @@ const configSchema = z.object({
FIRE_PDF_ASYNC_FORCE_TEAM_IDS: z.string().optional(),
FIRE_PDF_ASYNC_DISABLE_TEAM_IDS: z.string().optional(),
FIRE_PDF_ASYNC_ALLOW_REQUEST_OVERRIDE: z.stringbool().default(false),
+ // Large-PDF by-reference submits (30-256MB files uploaded to GCS and
+ // handed to fire-pdf via `input_gcs_uri`). This is an explicit on/off
+ // switch, not a percentage: no alternative engine exists at this size,
+ // so there is no cohort to sample "out" β€” only text-only degradation.
+ // FIRE_PDF_ENABLE remains the master switch for both paths.
+ FIRE_PDF_BY_REFERENCE_ENABLE: z.stringbool().default(true),
+ // Bucket that receives large-PDF inputs for by-reference async submits
+ // (fire-pdf reads them back via `input_gcs_uri`). fire-pdf only accepts
+ // URIs inside its own configured bucket + `inputs/` prefix, so this must
+ // match fire-pdf's FIRE_PDF_GCS_BUCKET. Upload failures (e.g. missing
+ // IAM grant) fall back to the pre-by-reference behavior for oversized
+ // files rather than failing the scrape.
+ FIRE_PDF_GCS_INPUT_BUCKET: z
+ .string()
+ .trim()
+ .min(1)
+ .default("firecrawl-pdf-pipeline"),
// RunPod
RUNPOD_MU_API_KEY: z.string().optional(),
diff --git a/apps/api/src/lib/gcs-pdf-cache.ts b/apps/api/src/lib/gcs-pdf-cache.ts
index 6cb865109..4f2465af5 100644
--- a/apps/api/src/lib/gcs-pdf-cache.ts
+++ b/apps/api/src/lib/gcs-pdf-cache.ts
@@ -30,8 +30,19 @@ export function createPdfCacheKey(pdfContent: string | Buffer): string {
return crypto.createHash("sha256").update(pdfContent).digest("hex");
}
+/** Cache addressing: historically the key is sha256 of the inline base64
+ * payload. Callers that never materialize the base64 (large PDFs submitted
+ * by GCS reference) pass a precomputed key instead β€” namespaced by the
+ * caller (e.g. `raw-<sha256-of-bytes>`) so the two keyspaces stay
+ * distinct. */
+export type PdfCacheKeyInput = string | { key: string };
+
+function resolvePdfCacheKey(input: PdfCacheKeyInput): string {
+ return typeof input === "string" ? createPdfCacheKey(input) : input.key;
+}
+
export async function savePdfResultToCache(
- pdfContent: string,
+ pdfContent: PdfCacheKeyInput,
result: CachedPdfResult,
provider: PdfCacheProvider = "runpod",
variant?: string,
@@ -42,7 +53,7 @@ export async function savePdfResultToCache(
}
const prefix = PROVIDER_PREFIXES[provider];
- const cacheKey = createPdfCacheKey(pdfContent);
+ const cacheKey = resolvePdfCacheKey(pdfContent);
const objectKey = variant ? `${cacheKey}-${variant}` : cacheKey;
const bucket = storage.bucket(config.GCS_BUCKET_NAME);
const blob = bucket.file(`${prefix}${objectKey}.json`);
@@ -89,7 +100,7 @@ export async function savePdfResultToCache(
}
export async function getPdfResultFromCache(
- pdfContent: string,
+ pdfContent: PdfCacheKeyInput,
provider: PdfCacheProvider = "runpod",
variant?: string,
): Promise<CachedPdfResult | null> {
@@ -99,7 +110,7 @@ export async function getPdfResultFromCache(
}
const prefix = PROVIDER_PREFIXES[provider];
- const cacheKey = createPdfCacheKey(pdfContent);
+ const cacheKey = resolvePdfCacheKey(pdfContent);
const objectKey = variant ? `${cacheKey}-${variant}` : cacheKey;
const bucket = storage.bucket(config.GCS_BUCKET_NAME);
const blob = bucket.file(`${prefix}${objectKey}.json`);
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts
index 03d351129..72452aa21 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts
@@ -11,6 +11,10 @@ import {
FirePdfAsyncFailure,
scrapePDFWithFirePDFAsync,
} from "../fire-pdf/async";
+import {
+ getPdfResultFromCache,
+ savePdfResultToCache,
+} from "../../../../../lib/gcs-pdf-cache";
import { config } from "../../../../../config";
// ── Fixtures ─────────────────────────────────────────────────────────────
@@ -1136,4 +1140,156 @@ describe("scrapePDFWithFirePDFAsync", () => {
expect(delta).toBeGreaterThanOrEqual(5_000 - 100);
expect(delta).toBeLessThanOrEqual(30 * 60 * 1_000);
});
+
+ describe("by-reference input (large PDFs)", () => {
+ const BY_REF = {
+ gcsUri: "gs://firecrawl-pdf-pipeline/inputs/scrape-id-test.pdf",
+ sha256: "ab".repeat(32),
+ sizeBytes: 200 * 1024 * 1024,
+ };
+
+ it("submits input_gcs_uri + input_sha256 instead of pdf_b64", async () => {
+ vi.mocked(getPdfResultFromCache).mockClear();
+ vi.mocked(savePdfResultToCache).mockClear();
+ const { fetchImpl, calls } = makeFetchFromSequence([
+ {
+ matchUrl: /\/jobs$/,
+ matchMethod: "POST",
+ response: {
+ status: 202,
+ body: { scrape_id: "scrape-id-test", status: "queued", lane: "xl" },
+ },
+ },
+ {
+ matchUrl: /\/jobs\/scrape-id-test$/,
+ matchMethod: "GET",
+ response: {
+ status: 200,
+ body: { scrape_id: "x", status: "done", pages_processed: 6543 },
+ },
+ },
+ {
+ matchUrl: /\/jobs\/scrape-id-test\/result$/,
+ matchMethod: "GET",
+ response: {
+ status: 200,
+ body: {
+ schema_version: 1,
+ markdown: "# big doc",
+ pages_processed: 6543,
+ failed_pages: null,
+ partial_pages: null,
+ },
+ },
+ },
+ ]);
+
+ const result = await scrapePDFWithFirePDFAsync(
+ makeMeta(),
+ BY_REF,
+ undefined,
+ 6543,
+ undefined,
+ { fetchImpl, fallbackImpl: vi.fn(), sleepImpl: noopSleep },
+ );
+
+ expect(result.markdown).toBe("# big doc");
+ // The by-reference LOOKUP happens at the call site before the input
+ // object is uploaded (a hit must skip the transfer, which has already
+ // happened by the time this function runs) β€” so no lookup here, only
+ // the save, addressed by the raw-byte sha namespaced apart from
+ // inline base64-keyed entries.
+ expect(vi.mocked(getPdfResultFromCache)).not.toHaveBeenCalled();
+ expect(vi.mocked(savePdfResultToCache)).toHaveBeenCalledWith(
+ { key: `raw-${BY_REF.sha256}` },
+ expect.anything(),
+ "firepdf",
+ undefined,
+ );
+ const body = calls[0].body as Record<string, unknown>;
+ expect(body.input_gcs_uri).toBe(BY_REF.gcsUri);
+ expect(body.input_sha256).toBe(BY_REF.sha256);
+ expect(body.pdf_b64).toBeUndefined();
+ expect((body.options as { pages_estimate?: number }).pages_estimate).toBe(
+ 6543,
+ );
+ });
+
+ it("clamps an explicit caller timeout to the 30min ceiling", async () => {
+ let submittedBody: any;
+ const fetchImpl: any = async (url: string, init: any) => {
+ if (/\/jobs$/.test(url) && (init?.method ?? "GET") === "POST") {
+ submittedBody = JSON.parse(init.body as string);
+ return jsonResp({
+ status: 200,
+ body: { scrape_id: "x", status: "done", pages_processed: 800 },
+ });
+ }
+ return jsonResp({
+ status: 200,
+ body: { markdown: "ok", pages_processed: 800 },
+ });
+ };
+ // Long documents need an explicit timeout: the no-budget default
+ // stays at 5 minutes because scrapeURLLoop kills no-timeout scrapes
+ // at 5 minutes regardless of the advertised FirePDF deadline. A
+ // timeout above MAX_DEADLINE_MS must be clamped to it.
+ const meta = makeMeta();
+ meta.abort.scrapeTimeout = vi.fn(() => 40 * 60 * 1_000);
+
+ await scrapePDFWithFirePDFAsync(
+ meta,
+ { ...BY_REF },
+ undefined,
+ 800,
+ undefined,
+ {
+ fetchImpl,
+ fallbackImpl: vi.fn(),
+ sleepImpl: noopSleep,
+ },
+ );
+
+ const delta = new Date(submittedBody.deadline_at).getTime() - Date.now();
+ expect(delta).toBeGreaterThan(29 * 60 * 1_000);
+ expect(delta).toBeLessThanOrEqual(30 * 60 * 1_000);
+ });
+
+ it("throws (never falls back) for by-reference under ZDR", async () => {
+ const fallback = vi.fn();
+ await expect(
+ scrapePDFWithFirePDFAsync(
+ makeMeta({
+ internalOptions: {
+ zeroDataRetention: true,
+ teamId: "team-x",
+ teamConcurrency: 12,
+ crawlId: undefined,
+ },
+ }),
+ { ...BY_REF },
+ undefined,
+ 100,
+ undefined,
+ { fetchImpl: vi.fn(), fallbackImpl: fallback, sleepImpl: noopSleep },
+ ),
+ ).rejects.toThrow(/zero data retention/);
+ expect(fallback).not.toHaveBeenCalled();
+ });
+
+ it("rejects a by-reference submit without a positive pages estimate", async () => {
+ for (const pagesEstimate of [undefined, 0, -1]) {
+ await expect(
+ scrapePDFWithFirePDFAsync(
+ makeMeta(),
+ { ...BY_REF },
+ undefined,
+ pagesEstimate,
+ undefined,
+ { fetchImpl: vi.fn(), fallbackImpl: vi.fn(), sleepImpl: noopSleep },
+ ),
+ ).rejects.toThrow(/pages estimate/);
+ }
+ });
+ });
});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
index 53e135bd6..6e680ec11 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
@@ -10,6 +10,7 @@ import { tryGetCached, maybeSaveResult } from "./cache";
import { firePdfAsyncTotalDurationSeconds } from "./metrics";
import { pollUntilTerminal } from "./poll";
import { fetchResult } from "./result";
+import type { FirePdfByReferenceInput } from "./by-reference";
import { FIRE_PDF_ASYNC_MIN_REMAINING_MS } from "./routing";
import { POLL_FLOOR_MS, POLL_TIMEOUT_BUFFER_MS } from "./schema";
import { submitJob, SubmitJobMayHaveBeenAcceptedError } from "./submit";
@@ -32,7 +33,11 @@ type FirePdfAsyncDeps = {
export async function scrapePDFWithFirePDFAsync(
meta: Meta,
- base64Content: string,
+ /** Inline base64 (string, the historical shape) or a pre-uploaded GCS
+ * reference for large files. By-reference has no sync fallback β€” the
+ * bytes don't fit fire-pdf's inline paths β€” so infra fallbacks below
+ * only apply to the string form. */
+ input: string | FirePdfByReferenceInput,
maxPages?: number,
pagesProcessed?: number,
mode?: PDFMode,
@@ -46,10 +51,18 @@ export async function scrapePDFWithFirePDFAsync(
const sleep = deps.sleepImpl ?? defaultSleep;
const now = deps.nowImpl ?? Date.now;
const random = deps.randomImpl ?? Math.random;
+ const base64Content = typeof input === "string" ? input : undefined;
// Async persists inputs and queue state, so ZDR is excluded until that
// lifecycle has an explicit delete-on-completion contract.
if (meta.internalOptions.zeroDataRetention) {
+ if (base64Content === undefined) {
+ // By-reference already persisted the input object; the routing layer
+ // must never send ZDR traffic here.
+ throw new Error(
+ "fire-pdf by-reference submit is not available under zero data retention",
+ );
+ }
return fallbackImpl(
meta,
base64Content,
@@ -62,16 +75,32 @@ export async function scrapePDFWithFirePDFAsync(
);
}
- const cached = await tryGetCached(
- meta,
- base64Content,
- mode,
- maxPages,
- pagesProcessed,
- includePageMarkdown,
- includeBlocks,
- pageMarkers,
- );
+ // Cache addressing: inline submits keep the historical key (sha256 of
+ // the base64 payload); by-reference submits use a `raw-` prefixed key
+ // over the raw-byte sha, so repeat scrapes of the same large document
+ // don't reprocess it. The two keyspaces are deliberately distinct β€” an
+ // inline and a by-reference parse of the same document do not share
+ // entries. The LOOKUP for by-reference happens at the call site BEFORE
+ // the input object is uploaded (a hit must skip the 30-256MB transfer,
+ // which has already happened by the time this function runs); only the
+ // inline path looks up here. Both paths save here.
+ const cacheInput =
+ typeof input === "string"
+ ? input
+ : { key: `raw-${input.sha256.toLowerCase()}` };
+ const cached =
+ typeof input === "string"
+ ? await tryGetCached(
+ meta,
+ cacheInput,
+ mode,
+ maxPages,
+ pagesProcessed,
+ includePageMarkdown,
+ includeBlocks,
+ pageMarkers,
+ )
+ : null;
if (cached) return cached;
meta.abort.throwIfAborted();
@@ -88,6 +117,11 @@ export async function scrapePDFWithFirePDFAsync(
if (!baseUrl) {
// Should be unreachable β€” call site checks this β€” but fall back rather
// than crash if a route somehow bypasses the gate.
+ if (base64Content === undefined) {
+ throw new Error(
+ "fire-pdf by-reference submit requires FIRE_PDF_BASE_URL",
+ );
+ }
return fallbackImpl(
meta,
base64Content,
@@ -102,6 +136,10 @@ export async function scrapePDFWithFirePDFAsync(
const overallStartedAt = now();
const submitTime = now();
+ // Note for large by-reference documents: the no-budget fallback inside
+ // computeDeadlineMs is 5 minutes because scrapeURLLoop kills no-timeout
+ // scrapes at 5 minutes anyway. Callers wanting the full multi-minute
+ // window for big documents must pass an explicit `timeout`.
const deadlineFromNow = computeDeadlineMs(remainingMs);
const deadlineAt = new Date(submitTime + deadlineFromNow).toISOString();
const pollingDeadline = submitTime + deadlineFromNow + POLL_TIMEOUT_BUFFER_MS;
@@ -125,7 +163,14 @@ export async function scrapePDFWithFirePDFAsync(
const submit = await submitJob({
meta,
baseUrl,
- base64Content,
+ input:
+ typeof input === "string"
+ ? { kind: "inline", base64Content: input }
+ : {
+ kind: "byReference",
+ gcsUri: input.gcsUri,
+ sha256: input.sha256,
+ },
maxPages,
pagesProcessed,
mode,
@@ -232,7 +277,7 @@ export async function scrapePDFWithFirePDFAsync(
await maybeSaveResult({
meta,
- base64Content,
+ base64Content: cacheInput,
mode,
maxPages,
includePageMarkdown,
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts
new file mode 100644
index 000000000..b6660ba6b
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts
@@ -0,0 +1,186 @@
+import { createHash } from "node:crypto";
+import { createReadStream } from "node:fs";
+import { Transform } from "node:stream";
+import { pipeline } from "node:stream/promises";
+import type { Meta } from "../../..";
+import { config } from "../../../../../config";
+import { storage } from "../../../../../lib/gcs-jobs";
+
+/** Input handle for a FirePDF async submit that travels by GCS reference
+ * instead of inline base64. Produced by {@link uploadPdfInputForFirePdf}. */
+export type FirePdfByReferenceInput = {
+ gcsUri: string;
+ sha256: string;
+ sizeBytes: number;
+};
+
+/** Uploading 256MB in-cluster to GCS is seconds; this bound exists so a
+ * stuck stream cannot hold a scrape slot indefinitely. */
+const UPLOAD_TIMEOUT_MS = 120_000;
+
+/**
+ * Whether by-reference FirePDF routing is configured and permitted for this
+ * request, judged from signals available before the file is downloaded.
+ * Both the download-size admission and the routing gate use this ONE
+ * predicate; the routing gate then adds the file-dependent conditions
+ * (size window, page count, MinerU diversion).
+ *
+ * ZDR is excluded because the by-reference input object persists in GCS.
+ * A forced FirePDF request (pages/blocks/markers) needs only the base URL,
+ * mirroring the inline path's rule; otherwise both the master switch and
+ * the by-reference switch must be on.
+ */
+export function byReferenceConfigured(
+ meta: Meta,
+ forceFirePdfRequested: boolean,
+): boolean {
+ return (
+ !meta.internalOptions.zeroDataRetention &&
+ !!config.FIRE_PDF_BASE_URL &&
+ (forceFirePdfRequested ||
+ (!!config.FIRE_PDF_ENABLE && config.FIRE_PDF_BY_REFERENCE_ENABLE))
+ );
+}
+
+/**
+ * Stream a downloaded PDF from its temp file into the fire-pdf input bucket
+ * so the async pipeline can fetch it by reference. Single pass: the sha-256
+ * (fire-pdf's idempotency identity) is computed through a transform inside
+ * the upload pipeline, so the file is read exactly once and never buffered.
+ * A timeout or scrape cancellation aborts the transfer itself (both streams
+ * are destroyed) β€” the caller may unlink the temp file as soon as this
+ * returns.
+ *
+ * Cleanup contract: the uploaded object is deliberately NOT deleted here or
+ * on job completion. fire-pdf's committed-retry replay depends on the input
+ * object outliving the job, and the bucket's prefix-scoped lifecycle policy
+ * (inputs/ deleted after 1 day, owned with the bucket in infra) is the
+ * cleanup mechanism.
+ *
+ * Returns null on any failure (missing bucket grant, timeout, transport):
+ * the caller falls back to the pre-by-reference behavior for oversized
+ * files instead of failing the scrape on infra misconfiguration.
+ */
+/** Streaming sha-256 of a file on disk. The by-reference cache identity β€”
+ * computed BEFORE any upload so a cache hit skips the 30-256MB transfer
+ * entirely. A disk read is orders of magnitude cheaper than the upload it
+ * can save. */
+export async function sha256OfFile(
+ path: string,
+ signal?: AbortSignal,
+): Promise<string> {
+ const hash = createHash("sha256");
+ for await (const chunk of createReadStream(path, { signal })) {
+ hash.update(chunk as Buffer);
+ }
+ return hash.digest("hex");
+}
+
+export async function uploadPdfInputForFirePdf(
+ meta: Meta,
+ tempFilePath: string,
+ sizeBytes: number,
+ opts?: {
+ /** sha-256 already computed over this exact file (e.g. for the
+ * pre-upload cache check) β€” skips the in-pipeline hash pass. */
+ precomputedSha256?: string;
+ },
+): Promise<FirePdfByReferenceInput | null> {
+ const bucketName = config.FIRE_PDF_GCS_INPUT_BUCKET;
+ // Scrape ids are UUIDv7 (time-ordered); a short hash prefix spreads the
+ // keys across GCS partitions, same convention as gcs-jobs.ts.
+ const keyPrefix = createHash("sha256")
+ .update(meta.id)
+ .digest("hex")
+ .slice(0, 8);
+ const objectKey = `inputs/${keyPrefix}-${meta.id}.pdf`;
+ const startedAt = Date.now();
+ try {
+ // Both the hash pass and the upload stop on scrape cancellation as
+ // well as the local timeout β€” a cancelled request must not keep
+ // reading and uploading hundreds of MB it can no longer use.
+ const timeoutAbort = new AbortController();
+ const signal = AbortSignal.any([
+ timeoutAbort.signal,
+ meta.abort.asSignal(),
+ ]);
+ // A first asSignal() call AFTER cancellation returns a signal whose
+ // abort listeners were attached post-abort and never fire β€” check the
+ // manager directly before starting the timer or any stream.
+ meta.abort.throwIfAborted();
+ const timer = setTimeout(
+ () =>
+ timeoutAbort.abort(
+ new Error(`GCS input upload timed out after ${UPLOAD_TIMEOUT_MS}ms`),
+ ),
+ UPLOAD_TIMEOUT_MS,
+ );
+ const hash =
+ opts?.precomputedSha256 === undefined ? createHash("sha256") : null;
+ try {
+ // Stream construction lives inside the timer-owning try: a
+ // synchronous createWriteStream throw must still clear the timeout.
+ const writeStream = storage
+ .bucket(bucketName)
+ .file(objectKey)
+ .createWriteStream({
+ resumable: true,
+ metadata: {
+ contentType: "application/pdf",
+ metadata: { scrape_id: meta.id, source: "firecrawl" },
+ },
+ });
+ if (hash) {
+ const hashThrough = new Transform({
+ transform(chunk, _encoding, callback) {
+ hash.update(chunk as Buffer);
+ callback(null, chunk);
+ },
+ });
+ await pipeline(
+ createReadStream(tempFilePath),
+ hashThrough,
+ writeStream,
+ {
+ signal,
+ },
+ );
+ } else {
+ await pipeline(createReadStream(tempFilePath), writeStream, { signal });
+ }
+ } finally {
+ clearTimeout(timer);
+ }
+ const sha256 = opts?.precomputedSha256 ?? hash!.digest("hex");
+ meta.logger.info("Uploaded large PDF for by-reference FirePDF submit", {
+ method: "scrapePDF/firePdfByReference",
+ event: "fire_pdf_by_reference_uploaded",
+ scrape_id: meta.id,
+ size_bytes: sizeBytes,
+ duration_ms: Date.now() - startedAt,
+ gcs_uri: `gs://${bucketName}/${objectKey}`,
+ });
+ return {
+ gcsUri: `gs://${bucketName}/${objectKey}`,
+ sha256,
+ sizeBytes,
+ };
+ } catch (error) {
+ // A cancelled scrape propagates as an abort, not a fallback β€” the
+ // legacy chain must not keep processing a request nobody is waiting on.
+ meta.abort.throwIfAborted();
+ meta.logger.warn(
+ "Large-PDF GCS input upload failed; falling back to legacy handling",
+ {
+ method: "scrapePDF/firePdfByReference",
+ event: "fire_pdf_by_reference_upload_failed",
+ scrape_id: meta.id,
+ team_id: meta.internalOptions.teamId,
+ size_bytes: sizeBytes,
+ bucket: bucketName,
+ error,
+ },
+ );
+ return null;
+ }
+}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts
index d62fc723e..9c963edad 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts
@@ -4,6 +4,7 @@ import type { PDFProcessorResult } from "../types";
import {
getPdfResultFromCache,
savePdfResultToCache,
+ type PdfCacheKeyInput,
} from "../../../../../lib/gcs-pdf-cache";
import { firePdfBlockPagesSchema } from "./schema";
@@ -143,7 +144,7 @@ export function cacheKeyShape(
export async function tryGetCached(
meta: Meta,
- base64Content: string,
+ base64Content: PdfCacheKeyInput,
mode: PDFMode | undefined,
maxPages: number | undefined,
pagesProcessed: number | undefined,
@@ -205,7 +206,7 @@ export async function tryGetCached(
export async function maybeSaveResult(args: {
meta: Meta;
- base64Content: string;
+ base64Content: PdfCacheKeyInput;
mode: PDFMode | undefined;
maxPages: number | undefined;
includePageMarkdown: boolean;
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
index 716cc4791..8fa4653a2 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
@@ -12,10 +12,18 @@ type SubmitOutcome = {
alreadyDone: boolean;
};
+/** How the PDF bytes reach fire-pdf: inline base64 for small files, or a
+ * GCS reference (pre-uploaded to fire-pdf's input bucket) for large ones.
+ * By-reference submits require a positive `pages_estimate` β€” fire-pdf has
+ * no bytes to probe at admission time. */
+type FirePdfSubmitInput =
+ | { kind: "inline"; base64Content: string }
+ | { kind: "byReference"; gcsUri: string; sha256: string };
+
type SubmitArgs = {
meta: Meta;
baseUrl: string;
- base64Content: string;
+ input: FirePdfSubmitInput;
maxPages: number | undefined;
pagesProcessed: number | undefined;
mode: PDFMode | undefined;
@@ -57,7 +65,7 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {
const {
meta,
baseUrl,
- base64Content,
+ input,
maxPages,
pagesProcessed,
mode,
@@ -70,8 +78,22 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {
} = args;
const scrapeId = meta.id;
+ if (
+ input.kind === "byReference" &&
+ (pagesProcessed === undefined || pagesProcessed <= 0)
+ ) {
+ // fire-pdf rejects by-reference submits without a positive
+ // pages_estimate (400 missing_pages_estimate); fail here with the
+ // clearer local error instead of a wire round-trip.
+ throw new Error(
+ "fire-pdf by-reference submit requires a positive pages estimate",
+ );
+ }
+
const body = {
- pdf_b64: base64Content,
+ ...(input.kind === "inline"
+ ? { pdf_b64: input.base64Content }
+ : { input_gcs_uri: input.gcsUri, input_sha256: input.sha256 }),
scrape_id: scrapeId,
source: "firecrawl" as const,
zdr: false as const,
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts
index 594856a1b..0d0f30cb0 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts
@@ -39,9 +39,14 @@ export function nextPollDelay(
}
export function computeDeadlineMs(scrapeTimeoutMs: number | undefined): number {
- // 5min default when there's no scrape budget (CLI/tests). Routing rejects
- // budgets that are too short; only cap the upper bound here so we never
- // advertise more time to FirePDF than the caller actually has.
+ // 5min default when there's no scrape budget. This matches the usable
+ // window exactly: scrapeURLLoop races every engine against
+ // `scrapeTimeout() ?? 300000`, so a no-timeout scrape dies at 5 minutes
+ // regardless of what we advertise to FirePDF. Long documents (large
+ // by-reference PDFs especially) need an explicit caller `timeout`, which
+ // unlocks up to MAX_DEADLINE_MS here and extends the loop's race alike.
+ // Only cap the upper bound so we never advertise more time to FirePDF
+ // than the caller actually has.
const fallback = 5 * 60 * 1_000;
const candidate = scrapeTimeoutMs ?? fallback;
return Math.min(MAX_DEADLINE_MS, candidate);
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index 9c6536dbf..b9d35ed58 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -11,7 +11,7 @@ import {
RemoveFeatureError,
EngineUnsuccessfulError,
} from "../../error";
-import { open, readFile, unlink } from "node:fs/promises";
+import { open, readFile, stat, unlink } from "node:fs/promises";
import type { Response } from "undici";
import { AbortManagerThrownError } from "../../lib/abortManager";
import {
@@ -26,6 +26,8 @@ import {
import type { PDFMode } from "../../../../controllers/v2/types";
import { processPdf, detectPdf } from "@mendable/firecrawl-rs";
import {
+ FIRE_PDF_BY_REFERENCE_MAX_FILE_SIZE,
+ FIRE_PDF_INLINE_HARD_MAX_FILE_SIZE,
FIRE_PDF_MAX_FILE_SIZE,
MAX_FILE_SIZE,
MILLISECONDS_PER_PAGE,
@@ -40,6 +42,12 @@ import { withSpan, setSpanAttributes } from "../../../../lib/otel-tracer";
import { scrapePDFWithRunPodMU } from "./runpodMU";
import { reconcilePageCountWithFirePdf, scrapePDFWithFirePDF } from "./firePDF";
import { scrapePDFWithFirePDFAsync } from "./fire-pdf/async";
+import {
+ byReferenceConfigured,
+ sha256OfFile,
+ uploadPdfInputForFirePdf,
+} from "./fire-pdf/by-reference";
+import { cacheKeyShape, tryGetCached } from "./fire-pdf/cache";
import { decideFirePdfAsyncRoute } from "./fire-pdf/routing";
import { scrapePDFWithParsePDF } from "./pdfParse";
import { toPublicBlocks } from "./blocks";
@@ -139,6 +147,37 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
}
}
+ const forceFirePDF =
+ (!!meta.options.__forceFirePDF ||
+ includePageMarkdown ||
+ includeBlocks ||
+ pageMarkers) &&
+ !!config.FIRE_PDF_BASE_URL;
+
+ // The MinerU diversion draw happens BEFORE the download so the admission
+ // cap can honor it: a diverted request cannot use the by-reference route,
+ // so it must keep the historical download cap instead of pulling up to
+ // 256MB it would only hand to pdf-parse. Forced Fire PDF takes
+ // precedence β€” don't divert those requests.
+ const routeToMinerU =
+ !forceFirePDF &&
+ config.MINERU_PERCENT > 0 &&
+ Math.random() * 100 < config.MINERU_PERCENT;
+
+ // Only admit large downloads when the by-reference FirePDF path is even
+ // reachable for this request (same predicate the routing gate uses; the
+ // gate adds the signals that need the file first). Otherwise keep the
+ // historical cap β€” an oversized file would only burn bandwidth and temp
+ // disk to fall through to text-only extraction.
+ // Fast mode is excluded: its hard cost ceiling skips the whole OCR/
+ // FirePDF chain (skipOCR), so the by-reference route it would justify is
+ // unreachable and the raised admission would only buffer bytes for the
+ // native path.
+ const byReferenceReachable =
+ !routeToMinerU &&
+ mode !== "fast" &&
+ byReferenceConfigured(meta, forceFirePDF);
+
const { response, tempFilePath } =
meta.pdfPrefetch !== undefined && meta.pdfPrefetch !== null
? { response: meta.pdfPrefetch, tempFilePath: meta.pdfPrefetch.filePath }
@@ -150,7 +189,11 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
headers: meta.options.headers,
signal: meta.abort.asSignal(),
},
- PDF_DOWNLOAD_MAX_FILE_SIZE,
+ // Parse path streams to disk and can hand large files to FirePDF
+ // by GCS reference, so it admits more than the raw fetch path.
+ byReferenceReachable
+ ? FIRE_PDF_BY_REFERENCE_MAX_FILE_SIZE
+ : PDF_DOWNLOAD_MAX_FILE_SIZE,
);
try {
@@ -194,22 +237,9 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
let shadowIneligibleReason: string | null | undefined;
let shadowPagesNeedingOcr: number[] | undefined;
- const forceFirePDF =
- (!!meta.options.__forceFirePDF ||
- includePageMarkdown ||
- includeBlocks ||
- pageMarkers) &&
- !!config.FIRE_PDF_BASE_URL;
const rustEnabled = !!config.PDF_RUST_EXTRACT_ENABLE;
const logger = meta.logger.child({ method: "scrapePDF/processPdf" });
- // Route a percentage of traffic directly to MinerU, bypassing Rust extraction.
- // Forced Fire PDF takes precedence β€” don't divert those requests.
- const routeToMinerU =
- !forceFirePDF &&
- config.MINERU_PERCENT > 0 &&
- Math.random() * 100 < config.MINERU_PERCENT;
-
if (routeToMinerU) {
logger.info("Routing to MinerU via MINERU_PERCENT", {
mineruPercent: config.MINERU_PERCENT,
@@ -405,10 +435,199 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
// unless we explicitly routed to MinerU via MINERU_PERCENT.
const skipOCR =
rustEnabled && mode === "fast" && !routeToMinerU && !forceFirePDF;
+
+ // Large PDFs can't travel inline as base64 JSON (fire-pdf's body limit,
+ // V8 string ceilings, worker memory), so they go to the fire-pdf async
+ // pipeline by GCS reference β€” streamed from the temp file, never
+ // buffered. ZDR stays out: the by-reference input object persists in
+ // GCS. MinerU-diverted traffic keeps its route decision (MinerU can't
+ // take these sizes, so the legacy chain below just skips through).
+ // By-reference has its own explicit switch instead of riding
+ // FIRE_PDF_PERCENT: there is no alternative engine at this size, so a
+ // sampled-out cohort would only degrade to text-only extraction. The
+ // shared predicate matches the download-admission decision above; only
+ // the file-dependent conditions are added here.
if (!result && !skipOCR) {
- const pdfBuffer = await readFile(tempFilePath);
- const fileSizeBytes = pdfBuffer.length;
- const base64Content = pdfBuffer.toString("base64");
+ const fileSizeBytes = (await stat(tempFilePath)).size;
+ // mode !== "fast" mirrors the admission gate above: fast mode keeps
+ // its pre-by-reference behavior on every path (with Rust extraction
+ // disabled, skipOCR alone would not exclude it here).
+ const useFirePdfByReference =
+ !routeToMinerU &&
+ mode !== "fast" &&
+ byReferenceConfigured(meta, forceFirePDF) &&
+ fileSizeBytes >= FIRE_PDF_MAX_FILE_SIZE &&
+ fileSizeBytes <= FIRE_PDF_BY_REFERENCE_MAX_FILE_SIZE;
+
+ if (useFirePdfByReference) {
+ if (effectivePageCount <= 0) {
+ // fire-pdf can't probe pages without the bytes, so by-reference
+ // submits require our page count. Fall through to the legacy
+ // chain (status quo for oversized files).
+ meta.logger.warn(
+ "Large PDF has no page-count estimate; cannot submit by reference",
+ {
+ method: "scrapePDF",
+ event: "fire_pdf_by_reference_no_pages",
+ file_size_bytes: fileSizeBytes,
+ scrape_id: meta.id,
+ team_id: meta.internalOptions.teamId,
+ },
+ );
+ } else {
+ // Cache BEFORE upload: the raw-byte sha is the by-reference cache
+ // identity, and it must be checked before the 30-256MB transfer β€”
+ // a repeat scrape of the same document should cost one streamed
+ // disk read, not a full re-upload. scrapePDFWithFirePDFAsync
+ // deliberately skips the by-reference lookup for the same reason
+ // (it runs post-upload) and only saves. Uncacheable requests
+ // (fast mode / maxPages) skip the pre-hash entirely β€” it could
+ // never produce a hit, and the upload computes its own hash
+ // in-pipeline. A failed pre-hash falls through to the upload
+ // path (legacy fallback semantics), never errors the scrape.
+ const { cacheable: byRefCacheable } = cacheKeyShape(
+ mode,
+ maxPages,
+ includePageMarkdown,
+ includeBlocks,
+ pageMarkers,
+ );
+ let localSha256: string | undefined;
+ if (byRefCacheable) {
+ try {
+ localSha256 = await sha256OfFile(
+ tempFilePath,
+ meta.abort.asSignal(),
+ );
+ } catch (error) {
+ meta.logger.warn(
+ "Pre-upload hash of large PDF failed; continuing without cache lookup",
+ {
+ method: "scrapePDF/firePdfByReference",
+ error,
+ scrape_id: meta.id,
+ },
+ );
+ }
+ }
+ const cachedByRef = localSha256
+ ? await tryGetCached(
+ meta,
+ { key: `raw-${localSha256}` },
+ mode,
+ maxPages,
+ effectivePageCount,
+ includePageMarkdown,
+ includeBlocks,
+ pageMarkers,
+ )
+ : null;
+ // A scrape cancelled during the hash/lookup must not return a
+ // success out of the cache.
+ meta.abort.throwIfAborted();
+ if (cachedByRef) {
+ result = cachedByRef;
+ effectivePageCount = reconcilePageCountWithFirePdf(
+ effectivePageCount,
+ result,
+ );
+ }
+ const uploaded = result
+ ? null
+ : await uploadPdfInputForFirePdf(
+ meta,
+ tempFilePath,
+ fileSizeBytes,
+ {
+ precomputedSha256: localSha256,
+ },
+ );
+ if (uploaded) {
+ try {
+ result = await scrapePDFWithFirePDFAsync(
+ {
+ ...meta,
+ logger: meta.logger.child({
+ method: "scrapePDF/firePDFAsyncByReference",
+ }),
+ },
+ uploaded,
+ maxPages,
+ effectivePageCount,
+ mode,
+ undefined,
+ includePageMarkdown,
+ includeBlocks,
+ pageMarkers,
+ );
+ effectivePageCount = reconcilePageCountWithFirePdf(
+ effectivePageCount,
+ result,
+ );
+ } catch (error) {
+ // No inline retry exists at this size, and the legacy chain
+ // below would silently degrade a large document to text-only
+ // extraction. Surface the failure instead.
+ meta.logger.error(
+ "FirePDF by-reference scrape failed (no fallback at this size)",
+ {
+ method: "scrapePDF/firePDFAsyncByReference",
+ error,
+ file_size_bytes: fileSizeBytes,
+ scrape_id: meta.id,
+ team_id: meta.internalOptions.teamId,
+ },
+ );
+ throw error;
+ }
+ }
+ // Upload failure: fall through to the legacy chain β€” the
+ // oversized-skip warning below still fires, preserving the
+ // pre-by-reference behavior.
+ }
+ }
+ }
+
+ if (!result && !skipOCR) {
+ const fileSizeBytes = (await stat(tempFilePath)).size;
+ // Only materialize the base64 payload for engines that can accept it
+ // inline: FirePDF (<30MB, or forced up to fire-pdf's wire ceiling)
+ // and MinerU (<19MB). Files above that reach this chain only as
+ // fallthrough (ZDR, MinerU-diverted, by-reference failure) and go
+ // straight to pdf-parse, which reads from disk β€” buffering and
+ // base64-encoding hundreds of MB here would only burn worker memory.
+ // Inline bytes are only materialized when an engine that can accept
+ // them on THIS route and size actually exists: FirePDF inline (not
+ // MinerU-diverted; under its cap, or forced up to the wire ceiling)
+ // or RunPod MU (under its own cap). Everything else goes straight to
+ // disk-based pdf-parse without a wasted base64 pass.
+ const firePdfInlineUsable =
+ (forceFirePDF ||
+ (!routeToMinerU &&
+ !!config.FIRE_PDF_ENABLE &&
+ !!config.FIRE_PDF_BASE_URL)) &&
+ (fileSizeBytes < FIRE_PDF_MAX_FILE_SIZE ||
+ (forceFirePDF &&
+ fileSizeBytes <= FIRE_PDF_INLINE_HARD_MAX_FILE_SIZE));
+ const runpodMuUsable =
+ !forceFirePDF &&
+ fileSizeBytes < MAX_FILE_SIZE &&
+ !!config.RUNPOD_MU_API_KEY &&
+ !!config.RUNPOD_MU_POD_ID;
+ const inlineEligible = firePdfInlineUsable || runpodMuUsable;
+ const base64Content = inlineEligible
+ ? (await readFile(tempFilePath)).toString("base64")
+ : undefined;
+
+ if (!result && forceFirePDF && base64Content === undefined) {
+ // Forced FirePDF (pages/blocks/markers) with no viable transport:
+ // the file exceeds the inline ceiling and the by-reference path was
+ // unavailable (ZDR) or failed. Erroring beats returning an empty
+ // document as a 200.
+ throw new Error(
+ `PDF (${fileSizeBytes} bytes) exceeds the FirePDF inline ceiling and by-reference submission was unavailable`,
+ );
+ }
if (
!forceFirePDF &&
@@ -432,12 +651,13 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
// forceFirePDF always wins; skip percentage-based Fire PDF when
// we explicitly routed to MinerU via MINERU_PERCENT.
const useFirePDF =
- forceFirePDF ||
- (!routeToMinerU &&
- config.FIRE_PDF_ENABLE &&
- config.FIRE_PDF_BASE_URL &&
- fileSizeBytes < FIRE_PDF_MAX_FILE_SIZE &&
- Math.random() * 100 < config.FIRE_PDF_PERCENT);
+ base64Content !== undefined &&
+ (forceFirePDF ||
+ (!routeToMinerU &&
+ config.FIRE_PDF_ENABLE &&
+ config.FIRE_PDF_BASE_URL &&
+ fileSizeBytes < FIRE_PDF_MAX_FILE_SIZE &&
+ Math.random() * 100 < config.FIRE_PDF_PERCENT));
if (useFirePDF) {
// Async is a server-controlled cohort within traffic already selected
@@ -596,6 +816,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
!result &&
!forceFirePDF &&
fileSizeBytes < MAX_FILE_SIZE &&
+ base64Content !== undefined &&
config.RUNPOD_MU_API_KEY &&
config.RUNPOD_MU_POD_ID
) {
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts
index c021e134f..e886d5b69 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts
@@ -64,6 +64,21 @@ export type PdfMetadata = {
};
export const MAX_FILE_SIZE = 19 * 1024 * 1024; // 19MB
+/** Above this, FirePDF submits go by GCS reference (async /jobs with
+ * `input_gcs_uri`) instead of inline base64 β€” inline bodies are capped by
+ * fire-pdf's JSON body limit and V8 string ceilings. */
export const FIRE_PDF_MAX_FILE_SIZE = 30 * 1024 * 1024; // 30MB
+/** Absolute ceiling for inline (base64 JSON) FirePDF submits, forced or
+ * not: fire-pdf's 100MB body limit divided by base64 inflation is ~74MB
+ * raw; 70MB leaves margin for the JSON envelope. Beyond this the inline
+ * path is a guaranteed 413, so callers shouldn't even build the string. */
+export const FIRE_PDF_INLINE_HARD_MAX_FILE_SIZE = 70 * 1024 * 1024; // 70MB
+/** Ceiling for by-reference FirePDF submits, and therefore for parse-path
+ * downloads (which stream to disk and hand large files to FirePDF by GCS
+ * reference). Mirrors fire-pdf's FIRE_PDF_GCS_INPUT_MAX_BYTES default β€”
+ * raise the two together. */
+export const FIRE_PDF_BY_REFERENCE_MAX_FILE_SIZE = 256 * 1024 * 1024; // 256MB
+/** Raw (non-parsed) PDF fetches return the file base64'd inline in the
+ * response, so they keep a tighter cap than the parse path. */
export const PDF_DOWNLOAD_MAX_FILE_SIZE = 50 * 1024 * 1024; // 50MB
export const MILLISECONDS_PER_PAGE = 150;