Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

feat(pdf): accept fire-engine's large-PDF GCS handoff (#4411)

Abimael Martella7e2c5513e20
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 33df37ea6..456d768c0 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -359,6 +359,30 @@ const configSchema = z.object({
.trim()
.min(1)
.default("firecrawl-pdf-pipeline"),
+ // Bucket fire-engine uses for its large-PDF handoff (files too big to
+ // inline as base64 in its response). Acts as the allowlist for inbound
+ // `file.gcs_uri` references β€” objects outside it are never fetched or
+ // copied. Deliberately no default: this is a security-sensitive inbound
+ // allowlist, so consuming references requires explicit opt-in (set to
+ // fire-engine's GCS_PDF_BUCKET_NAME); unset disables the path.
+ FIRE_ENGINE_PDF_GCS_BUCKET: emptyStringAsUndefined(z.string().trim().min(1)),
+ // Large-PDF size policy, applied per team on every acquisition path
+ // (direct download, fire-engine handoff, by-reference submit) and sent to
+ // fire-engine as the per-request pdfMaxSize. The default applies to every
+ // team; ids on the allowlist get the privileged cap. Both are clamped to
+ // the 256MB architectural ceiling.
+ PDF_BY_REFERENCE_MAX_BYTES_DEFAULT: z.coerce
+ .number()
+ .int()
+ .positive()
+ .default(50 * 1024 * 1024),
+ PDF_BY_REFERENCE_MAX_BYTES_PRIVILEGED: z.coerce
+ .number()
+ .int()
+ .positive()
+ .default(200 * 1024 * 1024),
+ // Comma-separated team ids granted the privileged cap.
+ PDF_BY_REFERENCE_PRIVILEGED_TEAM_IDS: z.string().optional(),
// RunPod
RUNPOD_MU_API_KEY: z.string().optional(),
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
index 8f53690b0..d77b0711b 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
@@ -104,11 +104,21 @@ const successSchema = z.object({
.array()
.optional(),
- // chrome-cdp only -- file download handler
+ // chrome-cdp only -- file download handler. Small files arrive inline as
+ // base64 `content`; large PDFs arrive as a GCS reference instead
+ // (fire-engine uploads them to its handoff bucket rather than inlining
+ // hundreds of MB of base64 into this response). Exactly one of
+ // `content` / `gcs_uri` is expected.
file: z
.object({
name: z.string(),
- content: z.string(),
+ content: z.string().optional(),
+ gcs_uri: z.string().optional(),
+ sha256: z.string().optional(),
+ size_bytes: z.number().optional(),
+ })
+ .refine(f => (f.content !== undefined) !== (f.gcs_uri !== undefined), {
+ message: "file must carry exactly one of content or gcs_uri",
})
.optional()
.or(z.null()),
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
index 06a4d3151..75bc07799 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -28,6 +28,23 @@ import {
import * as Sentry from "@sentry/node";
import { gunzipSync } from "node:zlib";
import { specialtyScrapeCheck } from "../utils/specialtyHandler";
+import {
+ byReferenceReachableForRequest,
+ largePdfLimitBytes,
+} from "../pdf/fire-pdf/by-reference";
+import { PDF_DOWNLOAD_MAX_FILE_SIZE } from "../pdf/types";
+import { config } from "../../../../config";
+
+/** The handoff additionally requires the inbound allowlist bucket to be
+ * configured: without it, a reference fire-engine returned could never be
+ * consumed, so granting the raise would only turn large-PDF scrapes into
+ * prefetch failures. */
+function fireEngineHandoffEligible(meta: Meta): boolean {
+ return (
+ config.FIRE_ENGINE_PDF_GCS_BUCKET !== undefined &&
+ byReferenceReachableForRequest(meta)
+ );
+}
import { fireEngineDelete } from "./delete";
import { MockState } from "../../lib/mock";
import { getInnerJson } from "@mendable/firecrawl-rs";
@@ -205,6 +222,13 @@ async function performFireEngineScrape<
}),
status.responseHeaders,
status,
+ meta.abort.asSignal(),
+ // Handoff downloads only admit large files when the FirePDF
+ // by-reference route can actually take them, and only up to the
+ // requesting team's large-PDF limit.
+ fireEngineHandoffEligible(meta)
+ ? largePdfLimitBytes(meta)
+ : PDF_DOWNLOAD_MAX_FILE_SIZE,
);
}
@@ -217,7 +241,10 @@ async function performFireEngineScrape<
status.content = await getInnerJson(status.content);
}
- if (status.file && !wantsRawBase64) {
+ // Reference-shaped files (gcs_uri without content) belong to the
+ // specialty prefetch path above and never reach this inline-decode
+ // block; guard on `content` so one slipping through can't crash it.
+ if (status.file?.content !== undefined && !wantsRawBase64) {
const content = status.file.content;
delete status.file;
let buffer = Buffer.from(content, "base64");
@@ -451,6 +478,12 @@ export async function scrapeURLWithFireEngineChromeCDP(
!meta.internalOptions.zeroDataRetention &&
meta.internalOptions.saveScrapeResultToGCS,
zeroDataRetention: meta.internalOptions.zeroDataRetention,
+ // Team-scoped ceiling for fire-engine's large-PDF GCS handoff: without
+ // it fire-engine grants no raise and PDFs keep its inline cap, so the
+ // worker never captures bytes this team may not use.
+ ...(fireEngineHandoffEligible(meta)
+ ? { pdfMaxSize: largePdfLimitBytes(meta) }
+ : {}),
...(shouldAllowMedia ? { blockMedia: false } : {}),
...(forceNonRender ? { forceNonRender: true } : {}),
persistentStorage: meta.options.profile
@@ -528,6 +561,19 @@ export async function scrapeURLWithFireEngineChromeCDP(
x => x[0].toLowerCase() === "content-type",
) ?? [])[1] ?? undefined;
+ // A GCS-reference file cannot serve rawBase64 (the caller wants inline
+ // bytes). fire-engine never grants the large-PDF raise to rawBase64
+ // requests, so this is defense in depth with a clear error rather than
+ // a silent rawBase64: undefined.
+ if (
+ hasFormatOfType(meta.options.formats, "rawBase64") !== undefined &&
+ response.file &&
+ response.file.content === undefined &&
+ response.file.gcs_uri !== undefined
+ ) {
+ throw new UnsupportedFileError("File exceeds size limit");
+ }
+
return {
url: response.url ?? meta.url,
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
index 03f481702..6c9f32573 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
@@ -61,6 +61,9 @@ export type FireEngineScrapeRequestCommon = {
export type FireEngineScrapeRequestChromeCDP = {
engine: "chrome-cdp";
skipTlsVerification?: boolean;
+ /** Team-scoped ceiling (bytes) for fire-engine's large-PDF GCS handoff.
+ * Absent = fire-engine grants no raise and PDFs keep its inline cap. */
+ pdfMaxSize?: number;
actions?: InternalAction[];
blockMedia?: boolean;
/** Opt out of render-engine routing (blockMedia: false normally forces it). */
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/fireEngineHandoff.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/fireEngineHandoff.test.ts
new file mode 100644
index 000000000..9afca6216
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/fireEngineHandoff.test.ts
@@ -0,0 +1,297 @@
+const gcsFake = vi.hoisted(() => ({
+ meta: { size: "1024", generation: "123456789012345678" } as {
+ size: string;
+ generation?: string;
+ },
+ bytes: Buffer.alloc(1024, 7),
+ reads: [] as Array<{ bucket: string; key: string; generation?: string }>,
+ copies: [] as Array<{
+ srcBucket: string;
+ srcKey: string;
+ generation?: string;
+ destBucket: string;
+ destKey: string;
+ }>,
+ failRead: false,
+ reset() {
+ this.meta = { size: "1024", generation: "123456789012345678" };
+ this.bytes = Buffer.alloc(1024, 7);
+ this.reads = [];
+ this.copies = [];
+ this.failRead = false;
+ },
+}));
+
+vi.mock("../../../../../lib/gcs-jobs", async () => {
+ const { Readable, PassThrough } = await import("node:stream");
+ const makeFile = (
+ bucket: string,
+ key: string,
+ opts?: { generation?: string },
+ ) => ({
+ _bucket: bucket,
+ _key: key,
+ getMetadata: async () => [gcsFake.meta],
+ createReadStream: () => {
+ gcsFake.reads.push({ bucket, key, generation: opts?.generation });
+ if (gcsFake.failRead) {
+ return new Readable({
+ read() {
+ this.destroy(new Error("stream fail"));
+ },
+ });
+ }
+ return Readable.from([gcsFake.bytes]);
+ },
+ copy: async (dest: { _bucket: string; _key: string }) => {
+ gcsFake.copies.push({
+ srcBucket: bucket,
+ srcKey: key,
+ generation: opts?.generation,
+ destBucket: dest._bucket,
+ destKey: dest._key,
+ });
+ },
+ createWriteStream: () => {
+ const p = new PassThrough();
+ p.resume();
+ return p;
+ },
+ });
+ return {
+ storage: {
+ bucket: (bucket: string) => ({
+ file: (key: string, opts?: { generation?: string }) =>
+ makeFile(bucket, key, opts),
+ }),
+ },
+ };
+});
+
+import {
+ largePdfLimitBytes,
+ rewritePdfInputForFirePdf,
+} from "../fire-pdf/by-reference";
+import { downloadFireEngineGcsFile } from "../../utils/downloadGcsFile";
+import { config } from "../../../../../config";
+import { stat, readFile as readFileAsync } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+
+function makeMeta() {
+ const noopLogger: any = {
+ info: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ debug: vi.fn(),
+ child: vi.fn(function child() {
+ return noopLogger;
+ }),
+ };
+ return {
+ id: "scrape-id-handoff",
+ logger: noopLogger,
+ abort: {
+ throwIfAborted: vi.fn(),
+ asSignal: vi.fn(() => new AbortController().signal),
+ scrapeTimeout: vi.fn(() => undefined),
+ },
+ internalOptions: { teamId: "team-x" },
+ } as any;
+}
+
+describe("fire-engine GCS handoff (bucket allowlists)", () => {
+ // The inbound allowlist is opt-in: configure it for these tests so the
+ // rejection cases exercise the bucket comparison, not the unset guard.
+ const ORIGINAL_BUCKET = config.FIRE_ENGINE_PDF_GCS_BUCKET;
+ beforeAll(() => {
+ (config as any).FIRE_ENGINE_PDF_GCS_BUCKET = "fire-engine-scrape-storage";
+ });
+ afterAll(() => {
+ (config as any).FIRE_ENGINE_PDF_GCS_BUCKET = ORIGINAL_BUCKET;
+ });
+
+ it("refuses every reference when the allowlist is unconfigured", async () => {
+ (config as any).FIRE_ENGINE_PDF_GCS_BUCKET = undefined;
+ try {
+ await expect(
+ downloadFireEngineGcsFile(
+ makeMeta().logger,
+ { uri: "gs://fire-engine-scrape-storage/pdf-handoff/x.pdf" },
+ "/tmp/never-written.pdf",
+ ),
+ ).resolves.toBeNull();
+ await expect(
+ rewritePdfInputForFirePdf(makeMeta(), {
+ uri: "gs://fire-engine-scrape-storage/pdf-handoff/x.pdf",
+ sha256: "ab".repeat(32),
+ sizeBytes: 1024,
+ }),
+ ).resolves.toBeNull();
+ } finally {
+ (config as any).FIRE_ENGINE_PDF_GCS_BUCKET = "fire-engine-scrape-storage";
+ }
+ });
+
+ it("rewrite refuses a source outside fire-engine's handoff bucket", async () => {
+ // Never copies out of an arbitrary bucket named by response data; the
+ // caller falls back to the streaming upload of the local temp file.
+ await expect(
+ rewritePdfInputForFirePdf(makeMeta(), {
+ uri: "gs://attacker-bucket/inputs/evil.pdf",
+ sha256: "ab".repeat(32),
+ sizeBytes: 1024,
+ }),
+ ).resolves.toBeNull();
+ });
+
+ it("rewrite refuses a malformed uri", async () => {
+ await expect(
+ rewritePdfInputForFirePdf(makeMeta(), {
+ uri: "https://storage.googleapis.com/not-a-gs-uri.pdf",
+ sha256: "ab".repeat(32),
+ sizeBytes: 1024,
+ }),
+ ).resolves.toBeNull();
+ });
+
+ it("download refuses references outside the handoff bucket", async () => {
+ const logger = makeMeta().logger;
+ await expect(
+ downloadFireEngineGcsFile(
+ logger,
+ { uri: "gs://some-other-bucket/pdf-handoff/x.pdf" },
+ "/tmp/never-written.pdf",
+ ),
+ ).resolves.toBeNull();
+ await expect(
+ downloadFireEngineGcsFile(
+ logger,
+ { uri: "not-a-uri" },
+ "/tmp/never-written.pdf",
+ ),
+ ).resolves.toBeNull();
+ });
+});
+
+describe("largePdfLimitBytes (team tiers)", () => {
+ const ORIGINAL = {
+ ids: config.PDF_BY_REFERENCE_PRIVILEGED_TEAM_IDS,
+ def: config.PDF_BY_REFERENCE_MAX_BYTES_DEFAULT,
+ priv: config.PDF_BY_REFERENCE_MAX_BYTES_PRIVILEGED,
+ };
+ afterEach(() => {
+ (config as any).PDF_BY_REFERENCE_PRIVILEGED_TEAM_IDS = ORIGINAL.ids;
+ (config as any).PDF_BY_REFERENCE_MAX_BYTES_DEFAULT = ORIGINAL.def;
+ (config as any).PDF_BY_REFERENCE_MAX_BYTES_PRIVILEGED = ORIGINAL.priv;
+ });
+
+ function metaForTeam(teamId?: string) {
+ return { internalOptions: { teamId } } as any;
+ }
+
+ it("returns the default cap (50MB) for unlisted teams", () => {
+ (config as any).PDF_BY_REFERENCE_PRIVILEGED_TEAM_IDS = "team-a, team-b";
+ expect(largePdfLimitBytes(metaForTeam("team-x"))).toBe(50 * 1024 * 1024);
+ expect(largePdfLimitBytes(metaForTeam(undefined))).toBe(50 * 1024 * 1024);
+ });
+
+ it("returns the privileged cap (200MB) for allowlisted teams", () => {
+ (config as any).PDF_BY_REFERENCE_PRIVILEGED_TEAM_IDS = "team-a, team-b";
+ expect(largePdfLimitBytes(metaForTeam("team-a"))).toBe(200 * 1024 * 1024);
+ expect(largePdfLimitBytes(metaForTeam("team-b"))).toBe(200 * 1024 * 1024);
+ });
+
+ it("clamps configured caps to the 256MB architectural ceiling", () => {
+ (config as any).PDF_BY_REFERENCE_PRIVILEGED_TEAM_IDS = "team-a";
+ (config as any).PDF_BY_REFERENCE_MAX_BYTES_PRIVILEGED = 999 * 1024 * 1024;
+ expect(largePdfLimitBytes(metaForTeam("team-a"))).toBe(256 * 1024 * 1024);
+ });
+});
+
+describe("fire-engine GCS handoff (positive paths, mocked storage)", () => {
+ const ORIGINAL_BUCKET = config.FIRE_ENGINE_PDF_GCS_BUCKET;
+ beforeAll(() => {
+ (config as any).FIRE_ENGINE_PDF_GCS_BUCKET = "fire-engine-scrape-storage";
+ });
+ afterAll(() => {
+ (config as any).FIRE_ENGINE_PDF_GCS_BUCKET = ORIGINAL_BUCKET;
+ });
+ beforeEach(() => gcsFake.reset());
+
+ it("streams a within-bucket reference to disk with a generation-pinned read", async () => {
+ const dest = path.join(tmpdir(), `handoff-test-${crypto.randomUUID()}.pdf`);
+ const res = await downloadFireEngineGcsFile(
+ makeMeta().logger,
+ { uri: "gs://fire-engine-scrape-storage/pdf-handoff/x.pdf" },
+ dest,
+ );
+ expect(res).toEqual({
+ sizeBytes: 1024,
+ generation: "123456789012345678",
+ });
+ expect((await stat(dest)).size).toBe(1024);
+ expect(await readFileAsync(dest)).toEqual(gcsFake.bytes);
+ // The read is pinned to the exact generation whose size was validated,
+ // carried as the SDK's string form (int64-safe).
+ expect(gcsFake.reads).toEqual([
+ {
+ bucket: "fire-engine-scrape-storage",
+ key: "pdf-handoff/x.pdf",
+ generation: "123456789012345678",
+ },
+ ]);
+ });
+
+ it("enforces the per-request size gate from metadata", async () => {
+ gcsFake.meta.size = String(300 * 1024 * 1024);
+ const res = await downloadFireEngineGcsFile(
+ makeMeta().logger,
+ { uri: "gs://fire-engine-scrape-storage/pdf-handoff/x.pdf" },
+ path.join(tmpdir(), `handoff-test-${crypto.randomUUID()}.pdf`),
+ undefined,
+ 50 * 1024 * 1024,
+ );
+ expect(res).toBeNull();
+ expect(gcsFake.reads).toEqual([]);
+ });
+
+ it("removes the partial temp file when the stream fails", async () => {
+ gcsFake.failRead = true;
+ const dest = path.join(tmpdir(), `handoff-test-${crypto.randomUUID()}.pdf`);
+ const res = await downloadFireEngineGcsFile(
+ makeMeta().logger,
+ { uri: "gs://fire-engine-scrape-storage/pdf-handoff/x.pdf" },
+ dest,
+ );
+ expect(res).toBeNull();
+ await expect(stat(dest)).rejects.toThrow();
+ });
+
+ it("rewrites into the fire-pdf input bucket with a generation-pinned copy", async () => {
+ const meta = makeMeta();
+ const res = await rewritePdfInputForFirePdf(meta, {
+ uri: "gs://fire-engine-scrape-storage/pdf-handoff/x.pdf",
+ sha256: "ab".repeat(32),
+ sizeBytes: 1024,
+ generation: "123456789012345678",
+ });
+ expect(res).toEqual({
+ gcsUri: expect.stringMatching(
+ /^gs:\/\/firecrawl-pdf-pipeline\/inputs\/[0-9a-f]{8}-scrape-id-handoff\.pdf$/,
+ ),
+ sha256: "ab".repeat(32),
+ sizeBytes: 1024,
+ });
+ expect(gcsFake.copies).toHaveLength(1);
+ expect(gcsFake.copies[0]).toMatchObject({
+ srcBucket: "fire-engine-scrape-storage",
+ srcKey: "pdf-handoff/x.pdf",
+ generation: "123456789012345678",
+ destBucket: "firecrawl-pdf-pipeline",
+ });
+ expect(gcsFake.copies[0].destKey).toMatch(
+ /^inputs\/[0-9a-f]{8}-scrape-id-handoff\.pdf$/,
+ );
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts
index b6660ba6b..5818ca67b 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts
@@ -4,7 +4,15 @@ import { Transform } from "node:stream";
import { pipeline } from "node:stream/promises";
import type { Meta } from "../../..";
import { config } from "../../../../../config";
+import {
+ getPDFBlocks,
+ getPDFMode,
+ getPDFPageMarkdown,
+ getPDFPageMarkers,
+} from "../../../../../controllers/v2/types";
+import { deterministicPercentage } from "./routing";
import { storage } from "../../../../../lib/gcs-jobs";
+import { FIRE_PDF_BY_REFERENCE_MAX_FILE_SIZE } from "../types";
/** Input handle for a FirePDF async submit that travels by GCS reference
* instead of inline base64. Produced by {@link uploadPdfInputForFirePdf}. */
@@ -18,19 +26,191 @@ export type FirePdfByReferenceInput = {
* stuck stream cannot hold a scrape slot indefinitely. */
const UPLOAD_TIMEOUT_MS = 120_000;
+/** Server-side rewrites are metadata-speed regardless of object size. */
+const REWRITE_TIMEOUT_MS = 30_000;
+
+function firePdfInputObjectKey(scrapeId: string, variant?: string): string {
+ // Scrape ids are UUIDv7 (time-ordered); a short hash prefix spreads the
+ // keys across GCS partitions, same convention as gcs-jobs.ts. The variant
+ // suffix keeps transports on distinct keys: a timed-out (but still
+ // running) rewrite must never race a fallback upload on the same object.
+ const keyPrefix = createHash("sha256")
+ .update(scrapeId)
+ .digest("hex")
+ .slice(0, 8);
+ return `inputs/${keyPrefix}-${scrapeId}${variant ? `-${variant}` : ""}.pdf`;
+}
+
+/**
+ * Server-side copy a fire-engine-uploaded PDF (the large-file GCS handoff)
+ * into the fire-pdf input bucket, so the bytes never transit this process
+ * at all. Requires the handoff to carry a sha256 (fire-pdf's idempotency
+ * identity) β€” without one the caller must fall back to
+ * {@link uploadPdfInputForFirePdf}, which computes it while streaming.
+ *
+ * Returns null on any failure; callers fall back to the streaming upload
+ * (the bytes are already on local disk for detection anyway).
+ */
+export async function rewritePdfInputForFirePdf(
+ meta: Meta,
+ source: {
+ uri: string;
+ sha256: string;
+ sizeBytes: number;
+ /** Generation validated (and read) by the prefetch download β€” the copy
+ * is pinned to it so a replaced object can never smuggle different
+ * bytes past the local size/sniff checks. Kept as the SDK's string
+ * representation: generations are int64 and must not be rounded
+ * through a JS number. */
+ generation?: string;
+ },
+): Promise<FirePdfByReferenceInput | null> {
+ const match = /^gs:\/\/([^/]+)\/(.+)$/.exec(source.uri);
+ if (!match || match[1] !== config.FIRE_ENGINE_PDF_GCS_BUCKET) {
+ // Only copy out of fire-engine's handoff bucket β€” never an arbitrary
+ // bucket named by response data.
+ return null;
+ }
+ const destBucket = config.FIRE_PDF_GCS_INPUT_BUCKET;
+ const destKey = firePdfInputObjectKey(meta.id);
+ const startedAt = Date.now();
+ try {
+ // Never start the server-side copy for an already-cancelled scrape β€”
+ // it would only create an orphaned input object.
+ meta.abort.throwIfAborted();
+ const destFile = storage.bucket(destBucket).file(destKey);
+ let timer: NodeJS.Timeout | undefined;
+ // copy() accepts no AbortSignal, so this only stops the wait (timeout or
+ // scrape cancellation); a late copy cannot clobber anything because the
+ // fallback upload uses a distinct object key.
+ const abortSignal = meta.abort.asSignal();
+ const timeout = new Promise<never>((_, reject) => {
+ timer = setTimeout(
+ () =>
+ reject(
+ new Error(`GCS rewrite timed out after ${REWRITE_TIMEOUT_MS}ms`),
+ ),
+ REWRITE_TIMEOUT_MS,
+ );
+ if (abortSignal.aborted) {
+ reject(abortSignal.reason ?? new Error("aborted"));
+ return;
+ }
+ abortSignal.addEventListener(
+ "abort",
+ () => reject(abortSignal.reason ?? new Error("aborted")),
+ { once: true },
+ );
+ });
+ try {
+ // The rewrite carries the source object's metadata (fire-engine sets
+ // contentType application/pdf at upload), so no overrides needed.
+ const sourceFile =
+ source.generation !== undefined
+ ? storage
+ .bucket(match[1])
+ .file(match[2], { generation: source.generation })
+ : storage.bucket(match[1]).file(match[2]);
+ await Promise.race([sourceFile.copy(destFile), timeout]);
+ } finally {
+ clearTimeout(timer);
+ }
+ meta.logger.info("Rewrote fire-engine PDF handoff into fire-pdf inputs", {
+ method: "scrapePDF/firePdfByReference",
+ event: "fire_pdf_by_reference_rewritten",
+ scrape_id: meta.id,
+ size_bytes: source.sizeBytes,
+ duration_ms: Date.now() - startedAt,
+ source_uri: source.uri,
+ gcs_uri: `gs://${destBucket}/${destKey}`,
+ });
+ return {
+ gcsUri: `gs://${destBucket}/${destKey}`,
+ sha256: source.sha256,
+ sizeBytes: source.sizeBytes,
+ };
+ } catch (error) {
+ meta.abort.throwIfAborted();
+ meta.logger.warn(
+ "GCS rewrite of fire-engine PDF handoff failed; falling back to streaming upload",
+ {
+ method: "scrapePDF/firePdfByReference",
+ event: "fire_pdf_by_reference_rewrite_failed",
+ scrape_id: meta.id,
+ source_uri: source.uri,
+ error,
+ },
+ );
+ return null;
+ }
+}
+
+/**
+ * The per-team large-PDF byte limit: the privileged cap for allowlisted
+ * team ids, the default cap for everyone else, both clamped to the 256MB
+ * architectural ceiling. Every acquisition path enforces this one number β€”
+ * the direct-download admission, the fire-engine handoff download, the
+ * by-reference routing gate, and (as pdfMaxSize) fire-engine's own capture
+ * ceiling, so no path can admit bytes another would reject.
+ */
+export function largePdfLimitBytes(meta: Meta): number {
+ const teamId = meta.internalOptions.teamId;
+ const privilegedIds = new Set(
+ (config.PDF_BY_REFERENCE_PRIVILEGED_TEAM_IDS ?? "")
+ .split(",")
+ .map(id => id.trim())
+ .filter(Boolean),
+ );
+ const raw =
+ teamId && privilegedIds.has(teamId)
+ ? config.PDF_BY_REFERENCE_MAX_BYTES_PRIVILEGED
+ : config.PDF_BY_REFERENCE_MAX_BYTES_DEFAULT;
+ // Config is schema-validated to positive integers; the clamp bounds both
+ // ends anyway so an invalid value can never reject every PDF or send
+ // fire-engine a nonsensical pdfMaxSize.
+ return Math.min(Math.max(raw, 1), FIRE_PDF_BY_REFERENCE_MAX_FILE_SIZE);
+}
+
+/** Whether this request is diverted to MinerU. Deterministic on the scrape
+ * id (same distribution as a random draw) precisely so every gate β€” the
+ * fire-engine handoff grant, the download admission, and the PDF engine's
+ * routing β€” sees the SAME verdict; a per-call random draw would let them
+ * drift. Keyed apart from the async-cohort hash. */
+export function mineruDiverted(meta: Meta): boolean {
+ return (
+ config.MINERU_PERCENT > 0 &&
+ deterministicPercentage(`mineru:${meta.id}`) < config.MINERU_PERCENT
+ );
+}
+
+/** The full request-level reachability check β€” byReferenceConfigured plus
+ * the parser options that force FirePDF, fast-mode exclusion (its cost
+ * ceiling skips the whole FirePDF chain), and the MinerU diversion. One
+ * definition shared by the PDF engine's download admission/routing gate
+ * AND the fire-engine handoff grant, so the gates can never drift. */
+export function byReferenceReachableForRequest(meta: Meta): boolean {
+ const forceRequested =
+ !!meta.options.__forceFirePDF ||
+ getPDFPageMarkdown(meta.options.parsers) ||
+ getPDFBlocks(meta.options.parsers) ||
+ getPDFPageMarkers(meta.options.parsers);
+ return (
+ getPDFMode(meta.options.parsers) !== "fast" &&
+ !(!forceRequested && mineruDiverted(meta)) &&
+ byReferenceConfigured(meta, forceRequested)
+ );
+}
+
/**
* Whether by-reference FirePDF routing is configured and permitted for this
* request, judged from signals available before the file is downloaded.
- * Both the download-size admission and the routing gate use this ONE
- * predicate; the routing gate then adds the file-dependent conditions
- * (size window, page count, MinerU diversion).
*
* ZDR is excluded because the by-reference input object persists in GCS.
* A forced FirePDF request (pages/blocks/markers) needs only the base URL,
* mirroring the inline path's rule; otherwise both the master switch and
* the by-reference switch must be on.
*/
-export function byReferenceConfigured(
+function byReferenceConfigured(
meta: Meta,
forceFirePdfRequested: boolean,
): boolean {
@@ -84,16 +264,13 @@ export async function uploadPdfInputForFirePdf(
/** sha-256 already computed over this exact file (e.g. for the
* pre-upload cache check) β€” skips the in-pipeline hash pass. */
precomputedSha256?: string;
+ /** Distinct object-key suffix β€” pass when another transport (e.g. a
+ * timed-out rewrite) may still be writing this scrape's default key. */
+ keyVariant?: string;
},
): Promise<FirePdfByReferenceInput | null> {
const bucketName = config.FIRE_PDF_GCS_INPUT_BUCKET;
- // Scrape ids are UUIDv7 (time-ordered); a short hash prefix spreads the
- // keys across GCS partitions, same convention as gcs-jobs.ts.
- const keyPrefix = createHash("sha256")
- .update(meta.id)
- .digest("hex")
- .slice(0, 8);
- const objectKey = `inputs/${keyPrefix}-${meta.id}.pdf`;
+ const objectKey = firePdfInputObjectKey(meta.id, opts?.keyVariant);
const startedAt = Date.now();
try {
// Both the hash pass and the upload stop on scrape cancellation as
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index b9d35ed58..bbeb7a0eb 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -10,6 +10,7 @@ import {
PDFPrefetchFailed,
RemoveFeatureError,
EngineUnsuccessfulError,
+ UnsupportedFileError,
} from "../../error";
import { open, readFile, stat, unlink } from "node:fs/promises";
import type { Response } from "undici";
@@ -43,7 +44,10 @@ import { scrapePDFWithRunPodMU } from "./runpodMU";
import { reconcilePageCountWithFirePdf, scrapePDFWithFirePDF } from "./firePDF";
import { scrapePDFWithFirePDFAsync } from "./fire-pdf/async";
import {
- byReferenceConfigured,
+ byReferenceReachableForRequest,
+ largePdfLimitBytes,
+ mineruDiverted,
+ rewritePdfInputForFirePdf,
sha256OfFile,
uploadPdfInputForFirePdf,
} from "./fire-pdf/by-reference";
@@ -95,6 +99,13 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
if (!shouldParse) {
if (meta.pdfPrefetch !== undefined && meta.pdfPrefetch !== null) {
+ // The raw path returns the file base64'd inline in the response, so it
+ // keeps the historical cap even when a large prefetch (fire-engine GCS
+ // handoff) materialized a bigger file on disk for the parse path.
+ const prefetchSize = (await stat(meta.pdfPrefetch.filePath)).size;
+ if (prefetchSize > PDF_DOWNLOAD_MAX_FILE_SIZE) {
+ throw new UnsupportedFileError("File exceeds size limit");
+ }
const content = (await readFile(meta.pdfPrefetch.filePath)).toString(
"base64",
);
@@ -154,29 +165,22 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
pageMarkers) &&
!!config.FIRE_PDF_BASE_URL;
- // The MinerU diversion draw happens BEFORE the download so the admission
- // cap can honor it: a diverted request cannot use the by-reference route,
- // so it must keep the historical download cap instead of pulling up to
- // 256MB it would only hand to pdf-parse. Forced Fire PDF takes
- // precedence β€” don't divert those requests.
- const routeToMinerU =
- !forceFirePDF &&
- config.MINERU_PERCENT > 0 &&
- Math.random() * 100 < config.MINERU_PERCENT;
+ // The MinerU diversion is deterministic on the scrape id (see
+ // mineruDiverted) so this routing verdict is BY CONSTRUCTION the same
+ // one inside byReferenceReachableForRequest() β€” and the same one
+ // fire-engine used when granting the handoff. Forced Fire PDF takes
+ // precedence and is never diverted.
+ const routeToMinerU = !forceFirePDF && mineruDiverted(meta);
// Only admit large downloads when the by-reference FirePDF path is even
// reachable for this request (same predicate the routing gate uses; the
// gate adds the signals that need the file first). Otherwise keep the
// historical cap β€” an oversized file would only burn bandwidth and temp
// disk to fall through to text-only extraction.
- // Fast mode is excluded: its hard cost ceiling skips the whole OCR/
- // FirePDF chain (skipOCR), so the by-reference route it would justify is
- // unreachable and the raised admission would only buffer bytes for the
- // native path.
- const byReferenceReachable =
- !routeToMinerU &&
- mode !== "fast" &&
- byReferenceConfigured(meta, forceFirePDF);
+ // The shared predicate covers fast-mode exclusion and the MinerU
+ // diversion too, so this is the same verdict fire-engine used when
+ // granting (or withholding) the handoff for this request.
+ const byReferenceReachable = byReferenceReachableForRequest(meta);
const { response, tempFilePath } =
meta.pdfPrefetch !== undefined && meta.pdfPrefetch !== null
@@ -190,9 +194,10 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
signal: meta.abort.asSignal(),
},
// Parse path streams to disk and can hand large files to FirePDF
- // by GCS reference, so it admits more than the raw fetch path.
+ // by GCS reference, so it admits more than the raw fetch path β€”
+ // up to the requesting team's large-PDF limit.
byReferenceReachable
- ? FIRE_PDF_BY_REFERENCE_MAX_FILE_SIZE
+ ? largePdfLimitBytes(meta)
: PDF_DOWNLOAD_MAX_FILE_SIZE,
);
@@ -449,15 +454,10 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
// the file-dependent conditions are added here.
if (!result && !skipOCR) {
const fileSizeBytes = (await stat(tempFilePath)).size;
- // mode !== "fast" mirrors the admission gate above: fast mode keeps
- // its pre-by-reference behavior on every path (with Rust extraction
- // disabled, skipOCR alone would not exclude it here).
const useFirePdfByReference =
- !routeToMinerU &&
- mode !== "fast" &&
- byReferenceConfigured(meta, forceFirePDF) &&
+ byReferenceReachable &&
fileSizeBytes >= FIRE_PDF_MAX_FILE_SIZE &&
- fileSizeBytes <= FIRE_PDF_BY_REFERENCE_MAX_FILE_SIZE;
+ fileSizeBytes <= largePdfLimitBytes(meta);
if (useFirePdfByReference) {
if (effectivePageCount <= 0) {
@@ -480,11 +480,16 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
// a repeat scrape of the same document should cost one streamed
// disk read, not a full re-upload. scrapePDFWithFirePDFAsync
// deliberately skips the by-reference lookup for the same reason
- // (it runs post-upload) and only saves. Uncacheable requests
- // (fast mode / maxPages) skip the pre-hash entirely β€” it could
- // never produce a hit, and the upload computes its own hash
- // in-pipeline. A failed pre-hash falls through to the upload
- // path (legacy fallback semantics), never errors the scrape.
+ // (it runs post-upload) and only saves.
+ //
+ // The local hash also verifies a fire-engine handoff before the
+ // server-side copy, so it is computed whenever a handoff carries
+ // a sha β€” even for uncacheable requests (maxPages), which skip
+ // only the cache LOOKUP. Requests with neither use skip the
+ // pre-hash entirely; the upload hashes in-pipeline. A failed
+ // pre-hash falls through to the upload path (legacy fallback
+ // semantics), never errors the scrape.
+ const handoff = meta.pdfPrefetch?.gcsReference;
const { cacheable: byRefCacheable } = cacheKeyShape(
mode,
maxPages,
@@ -493,7 +498,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
pageMarkers,
);
let localSha256: string | undefined;
- if (byRefCacheable) {
+ if (byRefCacheable || handoff?.sha256 !== undefined) {
try {
localSha256 = await sha256OfFile(
tempFilePath,
@@ -510,18 +515,19 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
);
}
}
- const cachedByRef = localSha256
- ? await tryGetCached(
- meta,
- { key: `raw-${localSha256}` },
- mode,
- maxPages,
- effectivePageCount,
- includePageMarkdown,
- includeBlocks,
- pageMarkers,
- )
- : null;
+ const cachedByRef =
+ byRefCacheable && localSha256
+ ? await tryGetCached(
+ meta,
+ { key: `raw-${localSha256}` },
+ mode,
+ maxPages,
+ effectivePageCount,
+ includePageMarkdown,
+ includeBlocks,
+ pageMarkers,
+ )
+ : null;
// A scrape cancelled during the hash/lookup must not return a
// success out of the cache.
meta.abort.throwIfAborted();
@@ -532,16 +538,59 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
result,
);
}
+ // On a miss: when fire-engine already handed the file off via
+ // GCS, a server-side rewrite moves it into the fire-pdf input
+ // bucket without the bytes transiting this process; otherwise
+ // (or if the rewrite fails) stream-upload the local temp file.
+ // The handoff hash becomes fire-pdf's idempotency identity, so it
+ // must match the raw-byte sha already computed for the cache
+ // check above (no second disk read needed); any mismatch falls
+ // back to the hashing upload.
+ const handoffShaMatches =
+ localSha256 !== undefined &&
+ handoff?.sha256 !== undefined &&
+ handoff.sha256.toLowerCase() === localSha256;
+ if (
+ localSha256 !== undefined &&
+ handoff?.sha256 !== undefined &&
+ !handoffShaMatches
+ ) {
+ meta.logger.warn(
+ "fire-engine handoff sha256 does not match local bytes; using streaming upload",
+ {
+ method: "scrapePDF/firePdfByReference",
+ event: "fire_pdf_handoff_sha_mismatch",
+ scrape_id: meta.id,
+ },
+ );
+ }
+ const rewriteEligible =
+ !result &&
+ handoffShaMatches &&
+ handoff!.sizeBytes === fileSizeBytes;
const uploaded = result
? null
- : await uploadPdfInputForFirePdf(
+ : ((rewriteEligible && handoff
+ ? await rewritePdfInputForFirePdf(meta, {
+ uri: handoff.uri,
+ // rewriteEligible implies handoffShaMatches implies defined
+ sha256: localSha256!,
+ sizeBytes: fileSizeBytes,
+ generation: handoff.generation,
+ })
+ : null) ??
+ // A distinct key when a rewrite was attempted: a timed-out
+ // copy may still complete and must never overwrite this
+ // upload.
+ (await uploadPdfInputForFirePdf(
meta,
tempFilePath,
fileSizeBytes,
{
+ keyVariant: rewriteEligible ? "s" : undefined,
precomputedSha256: localSha256,
},
- );
+ )));
if (uploaded) {
try {
result = await scrapePDFWithFirePDFAsync(
diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/downloadGcsFile.ts b/apps/api/src/scraper/scrapeURL/engines/utils/downloadGcsFile.ts
new file mode 100644
index 000000000..f97e35dc0
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/utils/downloadGcsFile.ts
@@ -0,0 +1,145 @@
+import { createWriteStream } from "node:fs";
+import { unlink } from "node:fs/promises";
+import { pipeline } from "node:stream/promises";
+import type { Logger } from "winston";
+import { config } from "../../../../config";
+import { storage } from "../../../../lib/gcs-jobs";
+import { FIRE_PDF_BY_REFERENCE_MAX_FILE_SIZE } from "../pdf/types";
+
+/** Downloading 256MB in-cluster from GCS is seconds; the bound exists so a
+ * stuck stream cannot hold a scrape slot indefinitely. */
+const DOWNLOAD_TIMEOUT_MS = 120_000;
+
+type FireEngineGcsFile = {
+ uri: string;
+ sha256?: string;
+ sizeBytes?: number;
+};
+
+/** Reject when `signal` fires while `promise` is pending β€” for GCS RPCs
+ * that accept no AbortSignal of their own. The RPC itself may briefly
+ * outlive the rejection; we stop waiting and never use its result. */
+async function raceWithSignal<T>(
+ promise: Promise<T>,
+ signal: AbortSignal,
+): Promise<T> {
+ if (signal.aborted) throw signal.reason ?? new Error("aborted");
+ let onAbort: (() => void) | undefined;
+ const abortP = new Promise<never>((_, reject) => {
+ onAbort = () => reject(signal.reason ?? new Error("aborted"));
+ signal.addEventListener("abort", onAbort, { once: true });
+ });
+ try {
+ return await Promise.race([promise, abortP]);
+ } finally {
+ if (onAbort) signal.removeEventListener("abort", onAbort);
+ }
+}
+
+function parseGcsUri(
+ uri: string,
+): { bucket: string; objectKey: string } | null {
+ const match = /^gs:\/\/([^/]+)\/(.+)$/.exec(uri);
+ if (!match) return null;
+ return { bucket: match[1], objectKey: match[2] };
+}
+
+/**
+ * Stream a fire-engine-uploaded file (large-PDF GCS handoff) from GCS to a
+ * local temp path. The URI comes from a fire-engine response; only objects
+ * inside fire-engine's configured handoff bucket are fetched β€” never an
+ * arbitrary bucket named by response data.
+ *
+ * Returns the byte size written, or null on any failure (wrong bucket,
+ * missing object, over-size, timeout) β€” callers treat null exactly like a
+ * prefetch that came back empty.
+ */
+export async function downloadFireEngineGcsFile(
+ logger: Logger,
+ file: FireEngineGcsFile,
+ destPath: string,
+ signal?: AbortSignal,
+ /** Admission cap for this request β€” pass the historical download cap when
+ * the FirePDF by-reference route is unreachable, so an unusable large
+ * handoff never consumes network and temp disk. Clamped to the
+ * by-reference ceiling. */
+ maxBytes?: number,
+): Promise<{ sizeBytes: number; generation?: string } | null> {
+ // An already-cancelled scrape must not proceed through any branch of
+ // this function β€” not even the cheap allowlist rejections, whose null
+ // returns would let the caller keep working on a dead request.
+ if (signal?.aborted) {
+ throw signal.reason ?? new Error("aborted");
+ }
+ const parsed = parseGcsUri(file.uri);
+ if (!parsed || parsed.bucket !== config.FIRE_ENGINE_PDF_GCS_BUCKET) {
+ logger.warn("fire-engine GCS file reference outside the handoff bucket", {
+ uri: file.uri,
+ expectedBucket: config.FIRE_ENGINE_PDF_GCS_BUCKET,
+ });
+ return null;
+ }
+ const effectiveMaxBytes = Math.min(
+ maxBytes ?? FIRE_PDF_BY_REFERENCE_MAX_FILE_SIZE,
+ FIRE_PDF_BY_REFERENCE_MAX_FILE_SIZE,
+ );
+
+ const timeoutAbort = new AbortController();
+ const combined = signal
+ ? AbortSignal.any([timeoutAbort.signal, signal])
+ : timeoutAbort.signal;
+ const timer = setTimeout(
+ () =>
+ timeoutAbort.abort(
+ new Error(`GCS file download timed out after ${DOWNLOAD_TIMEOUT_MS}ms`),
+ ),
+ DOWNLOAD_TIMEOUT_MS,
+ );
+ try {
+ const object = storage.bucket(parsed.bucket).file(parsed.objectKey);
+ const [metadata] = await raceWithSignal(object.getMetadata(), combined);
+ const sizeBytes = Number(metadata.size ?? file.sizeBytes ?? 0);
+ // Generations are int64 β€” keep the SDK's string form, never a JS number.
+ const generation =
+ metadata.generation !== undefined && metadata.generation !== null
+ ? String(metadata.generation)
+ : undefined;
+ if (!(sizeBytes > 0) || sizeBytes > effectiveMaxBytes) {
+ logger.warn("fire-engine GCS file reference has unusable size", {
+ uri: file.uri,
+ sizeBytes,
+ maxBytes: effectiveMaxBytes,
+ });
+ return null;
+ }
+
+ {
+ // Pin the read to the generation whose size was just validated, so a
+ // concurrent replacement of the object cannot bypass the size gate.
+ const pinned =
+ generation !== undefined
+ ? storage.bucket(parsed.bucket).file(parsed.objectKey, { generation })
+ : object;
+ await pipeline(pinned.createReadStream(), createWriteStream(destPath), {
+ signal: combined,
+ });
+ }
+ return { sizeBytes, generation };
+ } catch (error) {
+ // A failed stream may have written a partial file that no prefetch
+ // cleanup will ever see β€” remove it here.
+ await unlink(destPath).catch(() => {});
+ if (signal?.aborted) {
+ // Caller cancellation propagates as a cancellation, never as a
+ // synthetic prefetch failure.
+ throw signal.reason ?? error;
+ }
+ logger.warn("fire-engine GCS file download failed", {
+ uri: file.uri,
+ error,
+ });
+ return null;
+ } finally {
+ clearTimeout(timer);
+ }
+}
diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
index 749454c1e..b5415a887 100644
--- a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
@@ -6,6 +6,7 @@ import os from "os";
import { writeFile } from "fs/promises";
import { Meta } from "../..";
import { documentExtensionFromContentType } from "../../../../lib/document-formats";
+import { downloadFireEngineGcsFile } from "./downloadGcsFile";
async function feResToFilePrefetch(
logger: Logger,
@@ -13,8 +14,11 @@ async function feResToFilePrefetch(
fileExtension: string,
fileType: string,
contentType?: string,
+ signal?: AbortSignal,
+ maxFileBytes?: number,
): Promise<Meta["pdfPrefetch"] | Meta["documentPrefetch"]> {
- if (!feRes?.file) {
+ const file = feRes?.file;
+ if (!file || (file.content === undefined && file.gcs_uri === undefined)) {
logger.warn(`No file in ${fileType} prefetch`);
return null;
}
@@ -23,7 +27,33 @@ async function feResToFilePrefetch(
os.tmpdir(),
`tempFile-${crypto.randomUUID()}.${fileExtension}`,
);
- await writeFile(filePath, Buffer.from(feRes.file.content, "base64"));
+
+ let gcsReference: NonNullable<Meta["pdfPrefetch"]>["gcsReference"];
+ if (file.content !== undefined) {
+ await writeFile(filePath, Buffer.from(file.content, "base64"));
+ } else {
+ // Large-file handoff: fire-engine uploaded the bytes to GCS instead of
+ // inlining hundreds of MB of base64 through its response and job store.
+ // Materialize a local copy (magic-byte sniffing and page-count detection
+ // need bytes on disk) and keep the reference so the FirePDF by-reference
+ // path can server-side copy the object instead of re-uploading it.
+ const downloaded = await downloadFireEngineGcsFile(
+ logger,
+ { uri: file.gcs_uri!, sha256: file.sha256, sizeBytes: file.size_bytes },
+ filePath,
+ signal,
+ maxFileBytes,
+ );
+ if (downloaded === null) {
+ return null;
+ }
+ gcsReference = {
+ uri: file.gcs_uri!,
+ sha256: file.sha256,
+ sizeBytes: downloaded.sizeBytes,
+ generation: downloaded.generation,
+ };
+ }
return {
status: feRes.pageStatusCode,
@@ -31,14 +61,27 @@ async function feResToFilePrefetch(
filePath,
proxyUsed: feRes.usedMobileProxy ? "stealth" : "basic",
contentType,
+ // References are only produced for PDFs; the document prefetch shape
+ // does not carry one.
+ ...(fileType === "pdf" && gcsReference ? { gcsReference } : {}),
};
}
async function feResToPdfPrefetch(
logger: Logger,
feRes: FireEngineCheckStatusSuccess | undefined,
+ signal?: AbortSignal,
+ maxFileBytes?: number,
): Promise<Meta["pdfPrefetch"]> {
- return feResToFilePrefetch(logger, feRes, "pdf", "pdf");
+ return feResToFilePrefetch(
+ logger,
+ feRes,
+ "pdf",
+ "pdf",
+ undefined,
+ signal,
+ maxFileBytes,
+ );
}
async function feResToDocumentPrefetch(
@@ -57,11 +100,29 @@ export async function specialtyScrapeCheck(
logger: Logger,
headers: Record<string, string> | undefined,
feRes?: FireEngineCheckStatusSuccess,
+ /** Scrape abort signal β€” stops a large-PDF handoff download when the
+ * request has been cancelled or timed out. */
+ signal?: AbortSignal,
+ /** Admission cap for handoff downloads β€” the historical download cap when
+ * the FirePDF by-reference route is unreachable for this request, so an
+ * unusable large handoff never consumes network and temp disk. */
+ maxFileBytes?: number,
) {
const contentType = (Object.entries(headers ?? {}).find(
x => x[0].toLowerCase() === "content-type",
) ?? [])[1];
+ // A GCS reference is a PDF signal on its own β€” fire-engine only hands
+ // off verified PDFs β€” so it is handled before the content-type guard:
+ // some responses omit the header entirely, and the reference must still
+ // reach the FirePDF path.
+ if (feRes?.file?.gcs_uri !== undefined) {
+ throw new AddFeatureError(
+ ["pdf"],
+ await feResToPdfPrefetch(logger, feRes, signal, maxFileBytes),
+ );
+ }
+
if (!contentType) {
logger.warn("Failed to check contentType -- was not present in headers", {
headers,
@@ -89,10 +150,10 @@ export async function specialtyScrapeCheck(
// Legacy Office files (.doc, .xls) are OLE2/CFB files starting with D0 CF 11 E0 (base64: "0M8R4K")
if (isOctetStream) {
const isZipSignature =
- feRes?.file?.content.startsWith("UEsD") ||
+ feRes?.file?.content?.startsWith("UEsD") ||
feRes?.content.startsWith("PK");
const isOleSignature =
- feRes?.file?.content.startsWith("0M8R4K") ||
+ feRes?.file?.content?.startsWith("0M8R4K") ||
feRes?.content.startsWith("\xD0\xCF\x11\xE0");
if (isZipSignature) {
@@ -118,18 +179,24 @@ export async function specialtyScrapeCheck(
}
}
- // Check for PDF
+ // Check for PDF (references were already handled above the header guard).
if (isPdf) {
- throw new AddFeatureError(["pdf"], await feResToPdfPrefetch(logger, feRes));
+ throw new AddFeatureError(
+ ["pdf"],
+ await feResToPdfPrefetch(logger, feRes, signal, maxFileBytes),
+ );
}
// Check for octet-stream with PDF signature
if (
isOctetStream &&
- (feRes?.file?.content.startsWith("JVBERi0") ||
+ (feRes?.file?.content?.startsWith("JVBERi0") ||
feRes?.content.startsWith("%PDF-"))
) {
- throw new AddFeatureError(["pdf"], await feResToPdfPrefetch(logger, feRes));
+ throw new AddFeatureError(
+ ["pdf"],
+ await feResToPdfPrefetch(logger, feRes, signal, maxFileBytes),
+ );
}
// Reject unsupported binary content types (images, video, audio, archives, etc.)
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index c2792ab2c..a1a2ac5eb 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -151,6 +151,19 @@ export type Meta = {
status: number;
proxyUsed: "basic" | "stealth";
contentType?: string;
+ /** Set when fire-engine handed the file off by GCS reference (large
+ * PDFs): the object it uploaded, so the FirePDF by-reference path
+ * can server-side copy it instead of re-uploading the bytes. The
+ * local filePath is still materialized (sniffing and page-count
+ * detection need bytes on disk). */
+ gcsReference?: {
+ uri: string;
+ sha256?: string;
+ sizeBytes?: number;
+ /** int64 as the SDK's string form β€” never rounded through a JS
+ * number. */
+ generation?: string;
+ };
}
| null
| undefined; // undefined: no prefetch yet, null: prefetch came back empty