Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(pdf): accept fire-engine's large-PDF GCS handoff (#4411)
Abimael Martella7e2c5513e20
diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 33df37ea6..456d768c0 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -359,6 +359,30 @@ const configSchema = z.object({.trim().min(1).default("firecrawl-pdf-pipeline"),+ // Bucket fire-engine uses for its large-PDF handoff (files too big to+ // inline as base64 in its response). Acts as the allowlist for inbound+ // `file.gcs_uri` references β objects outside it are never fetched or+ // copied. Deliberately no default: this is a security-sensitive inbound+ // allowlist, so consuming references requires explicit opt-in (set to+ // fire-engine's GCS_PDF_BUCKET_NAME); unset disables the path.+ FIRE_ENGINE_PDF_GCS_BUCKET: emptyStringAsUndefined(z.string().trim().min(1)),+ // Large-PDF size policy, applied per team on every acquisition path+ // (direct download, fire-engine handoff, by-reference submit) and sent to+ // fire-engine as the per-request pdfMaxSize. The default applies to every+ // team; ids on the allowlist get the privileged cap. Both are clamped to+ // the 256MB architectural ceiling.+ PDF_BY_REFERENCE_MAX_BYTES_DEFAULT: z.coerce+ .number()+ .int()+ .positive()+ .default(50 * 1024 * 1024),+ PDF_BY_REFERENCE_MAX_BYTES_PRIVILEGED: z.coerce+ .number()+ .int()+ .positive()+ .default(200 * 1024 * 1024),+ // Comma-separated team ids granted the privileged cap.+ PDF_BY_REFERENCE_PRIVILEGED_TEAM_IDS: z.string().optional(),// RunPodRUNPOD_MU_API_KEY: z.string().optional(),diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.tsindex 8f53690b0..d77b0711b 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts@@ -104,11 +104,21 @@ const successSchema = z.object({.array().optional(),- // chrome-cdp only -- file download handler+ // chrome-cdp only -- file download handler. Small files arrive inline as+ // base64 `content`; large PDFs arrive as a GCS reference instead+ // (fire-engine uploads them to its handoff bucket rather than inlining+ // hundreds of MB of base64 into this response). Exactly one of+ // `content` / `gcs_uri` is expected.file: z.object({name: z.string(),- content: z.string(),+ content: z.string().optional(),+ gcs_uri: z.string().optional(),+ sha256: z.string().optional(),+ size_bytes: z.number().optional(),+ })+ .refine(f => (f.content !== undefined) !== (f.gcs_uri !== undefined), {+ message: "file must carry exactly one of content or gcs_uri",}).optional().or(z.null()),diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex 06a4d3151..75bc07799 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -28,6 +28,23 @@ import {import * as Sentry from "@sentry/node";import { gunzipSync } from "node:zlib";import { specialtyScrapeCheck } from "../utils/specialtyHandler";+import {+ byReferenceReachableForRequest,+ largePdfLimitBytes,+} from "../pdf/fire-pdf/by-reference";+import { PDF_DOWNLOAD_MAX_FILE_SIZE } from "../pdf/types";+import { config } from "../../../../config";++/** The handoff additionally requires the inbound allowlist bucket to be+ * configured: without it, a reference fire-engine returned could never be+ * consumed, so granting the raise would only turn large-PDF scrapes into+ * prefetch failures. */+function fireEngineHandoffEligible(meta: Meta): boolean {+ return (+ config.FIRE_ENGINE_PDF_GCS_BUCKET !== undefined &&+ byReferenceReachableForRequest(meta)+ );+}import { fireEngineDelete } from "./delete";import { MockState } from "../../lib/mock";import { getInnerJson } from "@mendable/firecrawl-rs";@@ -205,6 +222,13 @@ async function performFireEngineScrape<}),status.responseHeaders,status,+ meta.abort.asSignal(),+ // Handoff downloads only admit large files when the FirePDF+ // by-reference route can actually take them, and only up to the+ // requesting team's large-PDF limit.+ fireEngineHandoffEligible(meta)+ ? largePdfLimitBytes(meta)+ : PDF_DOWNLOAD_MAX_FILE_SIZE,);}@@ -217,7 +241,10 @@ async function performFireEngineScrape<status.content = await getInnerJson(status.content);}- if (status.file && !wantsRawBase64) {+ // Reference-shaped files (gcs_uri without content) belong to the+ // specialty prefetch path above and never reach this inline-decode+ // block; guard on `content` so one slipping through can't crash it.+ if (status.file?.content !== undefined && !wantsRawBase64) {const content = status.file.content;delete status.file;let buffer = Buffer.from(content, "base64");@@ -451,6 +478,12 @@ export async function scrapeURLWithFireEngineChromeCDP(!meta.internalOptions.zeroDataRetention &&meta.internalOptions.saveScrapeResultToGCS,zeroDataRetention: meta.internalOptions.zeroDataRetention,+ // Team-scoped ceiling for fire-engine's large-PDF GCS handoff: without+ // it fire-engine grants no raise and PDFs keep its inline cap, so the+ // worker never captures bytes this team may not use.+ ...(fireEngineHandoffEligible(meta)+ ? { pdfMaxSize: largePdfLimitBytes(meta) }+ : {}),...(shouldAllowMedia ? { blockMedia: false } : {}),...(forceNonRender ? { forceNonRender: true } : {}),persistentStorage: meta.options.profile@@ -528,6 +561,19 @@ export async function scrapeURLWithFireEngineChromeCDP(x => x[0].toLowerCase() === "content-type",) ?? [])[1] ?? undefined;+ // A GCS-reference file cannot serve rawBase64 (the caller wants inline+ // bytes). fire-engine never grants the large-PDF raise to rawBase64+ // requests, so this is defense in depth with a clear error rather than+ // a silent rawBase64: undefined.+ if (+ hasFormatOfType(meta.options.formats, "rawBase64") !== undefined &&+ response.file &&+ response.file.content === undefined &&+ response.file.gcs_uri !== undefined+ ) {+ throw new UnsupportedFileError("File exceeds size limit");+ }+return {url: response.url ?? meta.url,diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.tsindex 03f481702..6c9f32573 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts@@ -61,6 +61,9 @@ export type FireEngineScrapeRequestCommon = {export type FireEngineScrapeRequestChromeCDP = {engine: "chrome-cdp";skipTlsVerification?: boolean;+ /** Team-scoped ceiling (bytes) for fire-engine's large-PDF GCS handoff.+ * Absent = fire-engine grants no raise and PDFs keep its inline cap. */+ pdfMaxSize?: number;actions?: InternalAction[];blockMedia?: boolean;/** Opt out of render-engine routing (blockMedia: false normally forces it). */diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/fireEngineHandoff.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/fireEngineHandoff.test.tsnew file mode 100644index 000000000..9afca6216--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/fireEngineHandoff.test.ts@@ -0,0 +1,297 @@+const gcsFake = vi.hoisted(() => ({+ meta: { size: "1024", generation: "123456789012345678" } as {+ size: string;+ generation?: string;+ },+ bytes: Buffer.alloc(1024, 7),+ reads: [] as Array<{ bucket: string; key: string; generation?: string }>,+ copies: [] as Array<{+ srcBucket: string;+ srcKey: string;+ generation?: string;+ destBucket: string;+ destKey: string;+ }>,+ failRead: false,+ reset() {+ this.meta = { size: "1024", generation: "123456789012345678" };+ this.bytes = Buffer.alloc(1024, 7);+ this.reads = [];+ this.copies = [];+ this.failRead = false;+ },+}));++vi.mock("../../../../../lib/gcs-jobs", async () => {+ const { Readable, PassThrough } = await import("node:stream");+ const makeFile = (+ bucket: string,+ key: string,+ opts?: { generation?: string },+ ) => ({+ _bucket: bucket,+ _key: key,+ getMetadata: async () => [gcsFake.meta],+ createReadStream: () => {+ gcsFake.reads.push({ bucket, key, generation: opts?.generation });+ if (gcsFake.failRead) {+ return new Readable({+ read() {+ this.destroy(new Error("stream fail"));+ },+ });+ }+ return Readable.from([gcsFake.bytes]);+ },+ copy: async (dest: { _bucket: string; _key: string }) => {+ gcsFake.copies.push({+ srcBucket: bucket,+ srcKey: key,+ generation: opts?.generation,+ destBucket: dest._bucket,+ destKey: dest._key,+ });+ },+ createWriteStream: () => {+ const p = new PassThrough();+ p.resume();+ return p;+ },+ });+ return {+ storage: {+ bucket: (bucket: string) => ({+ file: (key: string, opts?: { generation?: string }) =>+ makeFile(bucket, key, opts),+ }),+ },+ };+});++import {+ largePdfLimitBytes,+ rewritePdfInputForFirePdf,+} from "../fire-pdf/by-reference";+import { downloadFireEngineGcsFile } from "../../utils/downloadGcsFile";+import { config } from "../../../../../config";+import { stat, readFile as readFileAsync } from "node:fs/promises";+import { tmpdir } from "node:os";+import path from "node:path";++function makeMeta() {+ const noopLogger: any = {+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ debug: vi.fn(),+ child: vi.fn(function child() {+ return noopLogger;+ }),+ };+ return {+ id: "scrape-id-handoff",+ logger: noopLogger,+ abort: {+ throwIfAborted: vi.fn(),+ asSignal: vi.fn(() => new AbortController().signal),+ scrapeTimeout: vi.fn(() => undefined),+ },+ internalOptions: { teamId: "team-x" },+ } as any;+}++describe("fire-engine GCS handoff (bucket allowlists)", () => {+ // The inbound allowlist is opt-in: configure it for these tests so the+ // rejection cases exercise the bucket comparison, not the unset guard.+ const ORIGINAL_BUCKET = config.FIRE_ENGINE_PDF_GCS_BUCKET;+ beforeAll(() => {+ (config as any).FIRE_ENGINE_PDF_GCS_BUCKET = "fire-engine-scrape-storage";+ });+ afterAll(() => {+ (config as any).FIRE_ENGINE_PDF_GCS_BUCKET = ORIGINAL_BUCKET;+ });++ it("refuses every reference when the allowlist is unconfigured", async () => {+ (config as any).FIRE_ENGINE_PDF_GCS_BUCKET = undefined;+ try {+ await expect(+ downloadFireEngineGcsFile(+ makeMeta().logger,+ { uri: "gs://fire-engine-scrape-storage/pdf-handoff/x.pdf" },+ "/tmp/never-written.pdf",+ ),+ ).resolves.toBeNull();+ await expect(+ rewritePdfInputForFirePdf(makeMeta(), {+ uri: "gs://fire-engine-scrape-storage/pdf-handoff/x.pdf",+ sha256: "ab".repeat(32),+ sizeBytes: 1024,+ }),+ ).resolves.toBeNull();+ } finally {+ (config as any).FIRE_ENGINE_PDF_GCS_BUCKET = "fire-engine-scrape-storage";+ }+ });++ it("rewrite refuses a source outside fire-engine's handoff bucket", async () => {+ // Never copies out of an arbitrary bucket named by response data; the+ // caller falls back to the streaming upload of the local temp file.+ await expect(+ rewritePdfInputForFirePdf(makeMeta(), {+ uri: "gs://attacker-bucket/inputs/evil.pdf",+ sha256: "ab".repeat(32),+ sizeBytes: 1024,+ }),+ ).resolves.toBeNull();+ });++ it("rewrite refuses a malformed uri", async () => {+ await expect(+ rewritePdfInputForFirePdf(makeMeta(), {+ uri: "https://storage.googleapis.com/not-a-gs-uri.pdf",+ sha256: "ab".repeat(32),+ sizeBytes: 1024,+ }),+ ).resolves.toBeNull();+ });++ it("download refuses references outside the handoff bucket", async () => {+ const logger = makeMeta().logger;+ await expect(+ downloadFireEngineGcsFile(+ logger,+ { uri: "gs://some-other-bucket/pdf-handoff/x.pdf" },+ "/tmp/never-written.pdf",+ ),+ ).resolves.toBeNull();+ await expect(+ downloadFireEngineGcsFile(+ logger,+ { uri: "not-a-uri" },+ "/tmp/never-written.pdf",+ ),+ ).resolves.toBeNull();+ });+});++describe("largePdfLimitBytes (team tiers)", () => {+ const ORIGINAL = {+ ids: config.PDF_BY_REFERENCE_PRIVILEGED_TEAM_IDS,+ def: config.PDF_BY_REFERENCE_MAX_BYTES_DEFAULT,+ priv: config.PDF_BY_REFERENCE_MAX_BYTES_PRIVILEGED,+ };+ afterEach(() => {+ (config as any).PDF_BY_REFERENCE_PRIVILEGED_TEAM_IDS = ORIGINAL.ids;+ (config as any).PDF_BY_REFERENCE_MAX_BYTES_DEFAULT = ORIGINAL.def;+ (config as any).PDF_BY_REFERENCE_MAX_BYTES_PRIVILEGED = ORIGINAL.priv;+ });++ function metaForTeam(teamId?: string) {+ return { internalOptions: { teamId } } as any;+ }++ it("returns the default cap (50MB) for unlisted teams", () => {+ (config as any).PDF_BY_REFERENCE_PRIVILEGED_TEAM_IDS = "team-a, team-b";+ expect(largePdfLimitBytes(metaForTeam("team-x"))).toBe(50 * 1024 * 1024);+ expect(largePdfLimitBytes(metaForTeam(undefined))).toBe(50 * 1024 * 1024);+ });++ it("returns the privileged cap (200MB) for allowlisted teams", () => {+ (config as any).PDF_BY_REFERENCE_PRIVILEGED_TEAM_IDS = "team-a, team-b";+ expect(largePdfLimitBytes(metaForTeam("team-a"))).toBe(200 * 1024 * 1024);+ expect(largePdfLimitBytes(metaForTeam("team-b"))).toBe(200 * 1024 * 1024);+ });++ it("clamps configured caps to the 256MB architectural ceiling", () => {+ (config as any).PDF_BY_REFERENCE_PRIVILEGED_TEAM_IDS = "team-a";+ (config as any).PDF_BY_REFERENCE_MAX_BYTES_PRIVILEGED = 999 * 1024 * 1024;+ expect(largePdfLimitBytes(metaForTeam("team-a"))).toBe(256 * 1024 * 1024);+ });+});++describe("fire-engine GCS handoff (positive paths, mocked storage)", () => {+ const ORIGINAL_BUCKET = config.FIRE_ENGINE_PDF_GCS_BUCKET;+ beforeAll(() => {+ (config as any).FIRE_ENGINE_PDF_GCS_BUCKET = "fire-engine-scrape-storage";+ });+ afterAll(() => {+ (config as any).FIRE_ENGINE_PDF_GCS_BUCKET = ORIGINAL_BUCKET;+ });+ beforeEach(() => gcsFake.reset());++ it("streams a within-bucket reference to disk with a generation-pinned read", async () => {+ const dest = path.join(tmpdir(), `handoff-test-${crypto.randomUUID()}.pdf`);+ const res = await downloadFireEngineGcsFile(+ makeMeta().logger,+ { uri: "gs://fire-engine-scrape-storage/pdf-handoff/x.pdf" },+ dest,+ );+ expect(res).toEqual({+ sizeBytes: 1024,+ generation: "123456789012345678",+ });+ expect((await stat(dest)).size).toBe(1024);+ expect(await readFileAsync(dest)).toEqual(gcsFake.bytes);+ // The read is pinned to the exact generation whose size was validated,+ // carried as the SDK's string form (int64-safe).+ expect(gcsFake.reads).toEqual([+ {+ bucket: "fire-engine-scrape-storage",+ key: "pdf-handoff/x.pdf",+ generation: "123456789012345678",+ },+ ]);+ });++ it("enforces the per-request size gate from metadata", async () => {+ gcsFake.meta.size = String(300 * 1024 * 1024);+ const res = await downloadFireEngineGcsFile(+ makeMeta().logger,+ { uri: "gs://fire-engine-scrape-storage/pdf-handoff/x.pdf" },+ path.join(tmpdir(), `handoff-test-${crypto.randomUUID()}.pdf`),+ undefined,+ 50 * 1024 * 1024,+ );+ expect(res).toBeNull();+ expect(gcsFake.reads).toEqual([]);+ });++ it("removes the partial temp file when the stream fails", async () => {+ gcsFake.failRead = true;+ const dest = path.join(tmpdir(), `handoff-test-${crypto.randomUUID()}.pdf`);+ const res = await downloadFireEngineGcsFile(+ makeMeta().logger,+ { uri: "gs://fire-engine-scrape-storage/pdf-handoff/x.pdf" },+ dest,+ );+ expect(res).toBeNull();+ await expect(stat(dest)).rejects.toThrow();+ });++ it("rewrites into the fire-pdf input bucket with a generation-pinned copy", async () => {+ const meta = makeMeta();+ const res = await rewritePdfInputForFirePdf(meta, {+ uri: "gs://fire-engine-scrape-storage/pdf-handoff/x.pdf",+ sha256: "ab".repeat(32),+ sizeBytes: 1024,+ generation: "123456789012345678",+ });+ expect(res).toEqual({+ gcsUri: expect.stringMatching(+ /^gs:\/\/firecrawl-pdf-pipeline\/inputs\/[0-9a-f]{8}-scrape-id-handoff\.pdf$/,+ ),+ sha256: "ab".repeat(32),+ sizeBytes: 1024,+ });+ expect(gcsFake.copies).toHaveLength(1);+ expect(gcsFake.copies[0]).toMatchObject({+ srcBucket: "fire-engine-scrape-storage",+ srcKey: "pdf-handoff/x.pdf",+ generation: "123456789012345678",+ destBucket: "firecrawl-pdf-pipeline",+ });+ expect(gcsFake.copies[0].destKey).toMatch(+ /^inputs\/[0-9a-f]{8}-scrape-id-handoff\.pdf$/,+ );+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.tsindex b6660ba6b..5818ca67b 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts@@ -4,7 +4,15 @@ import { Transform } from "node:stream";import { pipeline } from "node:stream/promises";import type { Meta } from "../../..";import { config } from "../../../../../config";+import {+ getPDFBlocks,+ getPDFMode,+ getPDFPageMarkdown,+ getPDFPageMarkers,+} from "../../../../../controllers/v2/types";+import { deterministicPercentage } from "./routing";import { storage } from "../../../../../lib/gcs-jobs";+import { FIRE_PDF_BY_REFERENCE_MAX_FILE_SIZE } from "../types";/** Input handle for a FirePDF async submit that travels by GCS reference* instead of inline base64. Produced by {@link uploadPdfInputForFirePdf}. */@@ -18,19 +26,191 @@ export type FirePdfByReferenceInput = {* stuck stream cannot hold a scrape slot indefinitely. */const UPLOAD_TIMEOUT_MS = 120_000;+/** Server-side rewrites are metadata-speed regardless of object size. */+const REWRITE_TIMEOUT_MS = 30_000;++function firePdfInputObjectKey(scrapeId: string, variant?: string): string {+ // Scrape ids are UUIDv7 (time-ordered); a short hash prefix spreads the+ // keys across GCS partitions, same convention as gcs-jobs.ts. The variant+ // suffix keeps transports on distinct keys: a timed-out (but still+ // running) rewrite must never race a fallback upload on the same object.+ const keyPrefix = createHash("sha256")+ .update(scrapeId)+ .digest("hex")+ .slice(0, 8);+ return `inputs/${keyPrefix}-${scrapeId}${variant ? `-${variant}` : ""}.pdf`;+}++/**+ * Server-side copy a fire-engine-uploaded PDF (the large-file GCS handoff)+ * into the fire-pdf input bucket, so the bytes never transit this process+ * at all. Requires the handoff to carry a sha256 (fire-pdf's idempotency+ * identity) β without one the caller must fall back to+ * {@link uploadPdfInputForFirePdf}, which computes it while streaming.+ *+ * Returns null on any failure; callers fall back to the streaming upload+ * (the bytes are already on local disk for detection anyway).+ */+export async function rewritePdfInputForFirePdf(+ meta: Meta,+ source: {+ uri: string;+ sha256: string;+ sizeBytes: number;+ /** Generation validated (and read) by the prefetch download β the copy+ * is pinned to it so a replaced object can never smuggle different+ * bytes past the local size/sniff checks. Kept as the SDK's string+ * representation: generations are int64 and must not be rounded+ * through a JS number. */+ generation?: string;+ },+): Promise<FirePdfByReferenceInput | null> {+ const match = /^gs:\/\/([^/]+)\/(.+)$/.exec(source.uri);+ if (!match || match[1] !== config.FIRE_ENGINE_PDF_GCS_BUCKET) {+ // Only copy out of fire-engine's handoff bucket β never an arbitrary+ // bucket named by response data.+ return null;+ }+ const destBucket = config.FIRE_PDF_GCS_INPUT_BUCKET;+ const destKey = firePdfInputObjectKey(meta.id);+ const startedAt = Date.now();+ try {+ // Never start the server-side copy for an already-cancelled scrape β+ // it would only create an orphaned input object.+ meta.abort.throwIfAborted();+ const destFile = storage.bucket(destBucket).file(destKey);+ let timer: NodeJS.Timeout | undefined;+ // copy() accepts no AbortSignal, so this only stops the wait (timeout or+ // scrape cancellation); a late copy cannot clobber anything because the+ // fallback upload uses a distinct object key.+ const abortSignal = meta.abort.asSignal();+ const timeout = new Promise<never>((_, reject) => {+ timer = setTimeout(+ () =>+ reject(+ new Error(`GCS rewrite timed out after ${REWRITE_TIMEOUT_MS}ms`),+ ),+ REWRITE_TIMEOUT_MS,+ );+ if (abortSignal.aborted) {+ reject(abortSignal.reason ?? new Error("aborted"));+ return;+ }+ abortSignal.addEventListener(+ "abort",+ () => reject(abortSignal.reason ?? new Error("aborted")),+ { once: true },+ );+ });+ try {+ // The rewrite carries the source object's metadata (fire-engine sets+ // contentType application/pdf at upload), so no overrides needed.+ const sourceFile =+ source.generation !== undefined+ ? storage+ .bucket(match[1])+ .file(match[2], { generation: source.generation })+ : storage.bucket(match[1]).file(match[2]);+ await Promise.race([sourceFile.copy(destFile), timeout]);+ } finally {+ clearTimeout(timer);+ }+ meta.logger.info("Rewrote fire-engine PDF handoff into fire-pdf inputs", {+ method: "scrapePDF/firePdfByReference",+ event: "fire_pdf_by_reference_rewritten",+ scrape_id: meta.id,+ size_bytes: source.sizeBytes,+ duration_ms: Date.now() - startedAt,+ source_uri: source.uri,+ gcs_uri: `gs://${destBucket}/${destKey}`,+ });+ return {+ gcsUri: `gs://${destBucket}/${destKey}`,+ sha256: source.sha256,+ sizeBytes: source.sizeBytes,+ };+ } catch (error) {+ meta.abort.throwIfAborted();+ meta.logger.warn(+ "GCS rewrite of fire-engine PDF handoff failed; falling back to streaming upload",+ {+ method: "scrapePDF/firePdfByReference",+ event: "fire_pdf_by_reference_rewrite_failed",+ scrape_id: meta.id,+ source_uri: source.uri,+ error,+ },+ );+ return null;+ }+}++/**+ * The per-team large-PDF byte limit: the privileged cap for allowlisted+ * team ids, the default cap for everyone else, both clamped to the 256MB+ * architectural ceiling. Every acquisition path enforces this one number β+ * the direct-download admission, the fire-engine handoff download, the+ * by-reference routing gate, and (as pdfMaxSize) fire-engine's own capture+ * ceiling, so no path can admit bytes another would reject.+ */+export function largePdfLimitBytes(meta: Meta): number {+ const teamId = meta.internalOptions.teamId;+ const privilegedIds = new Set(+ (config.PDF_BY_REFERENCE_PRIVILEGED_TEAM_IDS ?? "")+ .split(",")+ .map(id => id.trim())+ .filter(Boolean),+ );+ const raw =+ teamId && privilegedIds.has(teamId)+ ? config.PDF_BY_REFERENCE_MAX_BYTES_PRIVILEGED+ : config.PDF_BY_REFERENCE_MAX_BYTES_DEFAULT;+ // Config is schema-validated to positive integers; the clamp bounds both+ // ends anyway so an invalid value can never reject every PDF or send+ // fire-engine a nonsensical pdfMaxSize.+ return Math.min(Math.max(raw, 1), FIRE_PDF_BY_REFERENCE_MAX_FILE_SIZE);+}++/** Whether this request is diverted to MinerU. Deterministic on the scrape+ * id (same distribution as a random draw) precisely so every gate β the+ * fire-engine handoff grant, the download admission, and the PDF engine's+ * routing β sees the SAME verdict; a per-call random draw would let them+ * drift. Keyed apart from the async-cohort hash. */+export function mineruDiverted(meta: Meta): boolean {+ return (+ config.MINERU_PERCENT > 0 &&+ deterministicPercentage(`mineru:${meta.id}`) < config.MINERU_PERCENT+ );+}++/** The full request-level reachability check β byReferenceConfigured plus+ * the parser options that force FirePDF, fast-mode exclusion (its cost+ * ceiling skips the whole FirePDF chain), and the MinerU diversion. One+ * definition shared by the PDF engine's download admission/routing gate+ * AND the fire-engine handoff grant, so the gates can never drift. */+export function byReferenceReachableForRequest(meta: Meta): boolean {+ const forceRequested =+ !!meta.options.__forceFirePDF ||+ getPDFPageMarkdown(meta.options.parsers) ||+ getPDFBlocks(meta.options.parsers) ||+ getPDFPageMarkers(meta.options.parsers);+ return (+ getPDFMode(meta.options.parsers) !== "fast" &&+ !(!forceRequested && mineruDiverted(meta)) &&+ byReferenceConfigured(meta, forceRequested)+ );+}+/*** Whether by-reference FirePDF routing is configured and permitted for this* request, judged from signals available before the file is downloaded.- * Both the download-size admission and the routing gate use this ONE- * predicate; the routing gate then adds the file-dependent conditions- * (size window, page count, MinerU diversion).** ZDR is excluded because the by-reference input object persists in GCS.* A forced FirePDF request (pages/blocks/markers) needs only the base URL,* mirroring the inline path's rule; otherwise both the master switch and* the by-reference switch must be on.*/-export function byReferenceConfigured(+function byReferenceConfigured(meta: Meta,forceFirePdfRequested: boolean,): boolean {@@ -84,16 +264,13 @@ export async function uploadPdfInputForFirePdf(/** sha-256 already computed over this exact file (e.g. for the* pre-upload cache check) β skips the in-pipeline hash pass. */precomputedSha256?: string;+ /** Distinct object-key suffix β pass when another transport (e.g. a+ * timed-out rewrite) may still be writing this scrape's default key. */+ keyVariant?: string;},): Promise<FirePdfByReferenceInput | null> {const bucketName = config.FIRE_PDF_GCS_INPUT_BUCKET;- // Scrape ids are UUIDv7 (time-ordered); a short hash prefix spreads the- // keys across GCS partitions, same convention as gcs-jobs.ts.- const keyPrefix = createHash("sha256")- .update(meta.id)- .digest("hex")- .slice(0, 8);- const objectKey = `inputs/${keyPrefix}-${meta.id}.pdf`;+ const objectKey = firePdfInputObjectKey(meta.id, opts?.keyVariant);const startedAt = Date.now();try {// Both the hash pass and the upload stop on scrape cancellation asdiff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex b9d35ed58..bbeb7a0eb 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -10,6 +10,7 @@ import {PDFPrefetchFailed,RemoveFeatureError,EngineUnsuccessfulError,+ UnsupportedFileError,} from "../../error";import { open, readFile, stat, unlink } from "node:fs/promises";import type { Response } from "undici";@@ -43,7 +44,10 @@ import { scrapePDFWithRunPodMU } from "./runpodMU";import { reconcilePageCountWithFirePdf, scrapePDFWithFirePDF } from "./firePDF";import { scrapePDFWithFirePDFAsync } from "./fire-pdf/async";import {- byReferenceConfigured,+ byReferenceReachableForRequest,+ largePdfLimitBytes,+ mineruDiverted,+ rewritePdfInputForFirePdf,sha256OfFile,uploadPdfInputForFirePdf,} from "./fire-pdf/by-reference";@@ -95,6 +99,13 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {if (!shouldParse) {if (meta.pdfPrefetch !== undefined && meta.pdfPrefetch !== null) {+ // The raw path returns the file base64'd inline in the response, so it+ // keeps the historical cap even when a large prefetch (fire-engine GCS+ // handoff) materialized a bigger file on disk for the parse path.+ const prefetchSize = (await stat(meta.pdfPrefetch.filePath)).size;+ if (prefetchSize > PDF_DOWNLOAD_MAX_FILE_SIZE) {+ throw new UnsupportedFileError("File exceeds size limit");+ }const content = (await readFile(meta.pdfPrefetch.filePath)).toString("base64",);@@ -154,29 +165,22 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {pageMarkers) &&!!config.FIRE_PDF_BASE_URL;- // The MinerU diversion draw happens BEFORE the download so the admission- // cap can honor it: a diverted request cannot use the by-reference route,- // so it must keep the historical download cap instead of pulling up to- // 256MB it would only hand to pdf-parse. Forced Fire PDF takes- // precedence β don't divert those requests.- const routeToMinerU =- !forceFirePDF &&- config.MINERU_PERCENT > 0 &&- Math.random() * 100 < config.MINERU_PERCENT;+ // The MinerU diversion is deterministic on the scrape id (see+ // mineruDiverted) so this routing verdict is BY CONSTRUCTION the same+ // one inside byReferenceReachableForRequest() β and the same one+ // fire-engine used when granting the handoff. Forced Fire PDF takes+ // precedence and is never diverted.+ const routeToMinerU = !forceFirePDF && mineruDiverted(meta);// Only admit large downloads when the by-reference FirePDF path is even// reachable for this request (same predicate the routing gate uses; the// gate adds the signals that need the file first). Otherwise keep the// historical cap β an oversized file would only burn bandwidth and temp// disk to fall through to text-only extraction.- // Fast mode is excluded: its hard cost ceiling skips the whole OCR/- // FirePDF chain (skipOCR), so the by-reference route it would justify is- // unreachable and the raised admission would only buffer bytes for the- // native path.- const byReferenceReachable =- !routeToMinerU &&- mode !== "fast" &&- byReferenceConfigured(meta, forceFirePDF);+ // The shared predicate covers fast-mode exclusion and the MinerU+ // diversion too, so this is the same verdict fire-engine used when+ // granting (or withholding) the handoff for this request.+ const byReferenceReachable = byReferenceReachableForRequest(meta);const { response, tempFilePath } =meta.pdfPrefetch !== undefined && meta.pdfPrefetch !== null@@ -190,9 +194,10 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {signal: meta.abort.asSignal(),},// Parse path streams to disk and can hand large files to FirePDF- // by GCS reference, so it admits more than the raw fetch path.+ // by GCS reference, so it admits more than the raw fetch path β+ // up to the requesting team's large-PDF limit.byReferenceReachable- ? FIRE_PDF_BY_REFERENCE_MAX_FILE_SIZE+ ? largePdfLimitBytes(meta): PDF_DOWNLOAD_MAX_FILE_SIZE,);@@ -449,15 +454,10 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {// the file-dependent conditions are added here.if (!result && !skipOCR) {const fileSizeBytes = (await stat(tempFilePath)).size;- // mode !== "fast" mirrors the admission gate above: fast mode keeps- // its pre-by-reference behavior on every path (with Rust extraction- // disabled, skipOCR alone would not exclude it here).const useFirePdfByReference =- !routeToMinerU &&- mode !== "fast" &&- byReferenceConfigured(meta, forceFirePDF) &&+ byReferenceReachable &&fileSizeBytes >= FIRE_PDF_MAX_FILE_SIZE &&- fileSizeBytes <= FIRE_PDF_BY_REFERENCE_MAX_FILE_SIZE;+ fileSizeBytes <= largePdfLimitBytes(meta);if (useFirePdfByReference) {if (effectivePageCount <= 0) {@@ -480,11 +480,16 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {// a repeat scrape of the same document should cost one streamed// disk read, not a full re-upload. scrapePDFWithFirePDFAsync// deliberately skips the by-reference lookup for the same reason- // (it runs post-upload) and only saves. Uncacheable requests- // (fast mode / maxPages) skip the pre-hash entirely β it could- // never produce a hit, and the upload computes its own hash- // in-pipeline. A failed pre-hash falls through to the upload- // path (legacy fallback semantics), never errors the scrape.+ // (it runs post-upload) and only saves.+ //+ // The local hash also verifies a fire-engine handoff before the+ // server-side copy, so it is computed whenever a handoff carries+ // a sha β even for uncacheable requests (maxPages), which skip+ // only the cache LOOKUP. Requests with neither use skip the+ // pre-hash entirely; the upload hashes in-pipeline. A failed+ // pre-hash falls through to the upload path (legacy fallback+ // semantics), never errors the scrape.+ const handoff = meta.pdfPrefetch?.gcsReference;const { cacheable: byRefCacheable } = cacheKeyShape(mode,maxPages,@@ -493,7 +498,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {pageMarkers,);let localSha256: string | undefined;- if (byRefCacheable) {+ if (byRefCacheable || handoff?.sha256 !== undefined) {try {localSha256 = await sha256OfFile(tempFilePath,@@ -510,18 +515,19 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {);}}- const cachedByRef = localSha256- ? await tryGetCached(- meta,- { key: `raw-${localSha256}` },- mode,- maxPages,- effectivePageCount,- includePageMarkdown,- includeBlocks,- pageMarkers,- )- : null;+ const cachedByRef =+ byRefCacheable && localSha256+ ? await tryGetCached(+ meta,+ { key: `raw-${localSha256}` },+ mode,+ maxPages,+ effectivePageCount,+ includePageMarkdown,+ includeBlocks,+ pageMarkers,+ )+ : null;// A scrape cancelled during the hash/lookup must not return a// success out of the cache.meta.abort.throwIfAborted();@@ -532,16 +538,59 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {result,);}+ // On a miss: when fire-engine already handed the file off via+ // GCS, a server-side rewrite moves it into the fire-pdf input+ // bucket without the bytes transiting this process; otherwise+ // (or if the rewrite fails) stream-upload the local temp file.+ // The handoff hash becomes fire-pdf's idempotency identity, so it+ // must match the raw-byte sha already computed for the cache+ // check above (no second disk read needed); any mismatch falls+ // back to the hashing upload.+ const handoffShaMatches =+ localSha256 !== undefined &&+ handoff?.sha256 !== undefined &&+ handoff.sha256.toLowerCase() === localSha256;+ if (+ localSha256 !== undefined &&+ handoff?.sha256 !== undefined &&+ !handoffShaMatches+ ) {+ meta.logger.warn(+ "fire-engine handoff sha256 does not match local bytes; using streaming upload",+ {+ method: "scrapePDF/firePdfByReference",+ event: "fire_pdf_handoff_sha_mismatch",+ scrape_id: meta.id,+ },+ );+ }+ const rewriteEligible =+ !result &&+ handoffShaMatches &&+ handoff!.sizeBytes === fileSizeBytes;const uploaded = result? null- : await uploadPdfInputForFirePdf(+ : ((rewriteEligible && handoff+ ? await rewritePdfInputForFirePdf(meta, {+ uri: handoff.uri,+ // rewriteEligible implies handoffShaMatches implies defined+ sha256: localSha256!,+ sizeBytes: fileSizeBytes,+ generation: handoff.generation,+ })+ : null) ??+ // A distinct key when a rewrite was attempted: a timed-out+ // copy may still complete and must never overwrite this+ // upload.+ (await uploadPdfInputForFirePdf(meta,tempFilePath,fileSizeBytes,{+ keyVariant: rewriteEligible ? "s" : undefined,precomputedSha256: localSha256,},- );+ )));if (uploaded) {try {result = await scrapePDFWithFirePDFAsync(diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/downloadGcsFile.ts b/apps/api/src/scraper/scrapeURL/engines/utils/downloadGcsFile.tsnew file mode 100644index 000000000..f97e35dc0--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/utils/downloadGcsFile.ts@@ -0,0 +1,145 @@+import { createWriteStream } from "node:fs";+import { unlink } from "node:fs/promises";+import { pipeline } from "node:stream/promises";+import type { Logger } from "winston";+import { config } from "../../../../config";+import { storage } from "../../../../lib/gcs-jobs";+import { FIRE_PDF_BY_REFERENCE_MAX_FILE_SIZE } from "../pdf/types";++/** Downloading 256MB in-cluster from GCS is seconds; the bound exists so a+ * stuck stream cannot hold a scrape slot indefinitely. */+const DOWNLOAD_TIMEOUT_MS = 120_000;++type FireEngineGcsFile = {+ uri: string;+ sha256?: string;+ sizeBytes?: number;+};++/** Reject when `signal` fires while `promise` is pending β for GCS RPCs+ * that accept no AbortSignal of their own. The RPC itself may briefly+ * outlive the rejection; we stop waiting and never use its result. */+async function raceWithSignal<T>(+ promise: Promise<T>,+ signal: AbortSignal,+): Promise<T> {+ if (signal.aborted) throw signal.reason ?? new Error("aborted");+ let onAbort: (() => void) | undefined;+ const abortP = new Promise<never>((_, reject) => {+ onAbort = () => reject(signal.reason ?? new Error("aborted"));+ signal.addEventListener("abort", onAbort, { once: true });+ });+ try {+ return await Promise.race([promise, abortP]);+ } finally {+ if (onAbort) signal.removeEventListener("abort", onAbort);+ }+}++function parseGcsUri(+ uri: string,+): { bucket: string; objectKey: string } | null {+ const match = /^gs:\/\/([^/]+)\/(.+)$/.exec(uri);+ if (!match) return null;+ return { bucket: match[1], objectKey: match[2] };+}++/**+ * Stream a fire-engine-uploaded file (large-PDF GCS handoff) from GCS to a+ * local temp path. The URI comes from a fire-engine response; only objects+ * inside fire-engine's configured handoff bucket are fetched β never an+ * arbitrary bucket named by response data.+ *+ * Returns the byte size written, or null on any failure (wrong bucket,+ * missing object, over-size, timeout) β callers treat null exactly like a+ * prefetch that came back empty.+ */+export async function downloadFireEngineGcsFile(+ logger: Logger,+ file: FireEngineGcsFile,+ destPath: string,+ signal?: AbortSignal,+ /** Admission cap for this request β pass the historical download cap when+ * the FirePDF by-reference route is unreachable, so an unusable large+ * handoff never consumes network and temp disk. Clamped to the+ * by-reference ceiling. */+ maxBytes?: number,+): Promise<{ sizeBytes: number; generation?: string } | null> {+ // An already-cancelled scrape must not proceed through any branch of+ // this function β not even the cheap allowlist rejections, whose null+ // returns would let the caller keep working on a dead request.+ if (signal?.aborted) {+ throw signal.reason ?? new Error("aborted");+ }+ const parsed = parseGcsUri(file.uri);+ if (!parsed || parsed.bucket !== config.FIRE_ENGINE_PDF_GCS_BUCKET) {+ logger.warn("fire-engine GCS file reference outside the handoff bucket", {+ uri: file.uri,+ expectedBucket: config.FIRE_ENGINE_PDF_GCS_BUCKET,+ });+ return null;+ }+ const effectiveMaxBytes = Math.min(+ maxBytes ?? FIRE_PDF_BY_REFERENCE_MAX_FILE_SIZE,+ FIRE_PDF_BY_REFERENCE_MAX_FILE_SIZE,+ );++ const timeoutAbort = new AbortController();+ const combined = signal+ ? AbortSignal.any([timeoutAbort.signal, signal])+ : timeoutAbort.signal;+ const timer = setTimeout(+ () =>+ timeoutAbort.abort(+ new Error(`GCS file download timed out after ${DOWNLOAD_TIMEOUT_MS}ms`),+ ),+ DOWNLOAD_TIMEOUT_MS,+ );+ try {+ const object = storage.bucket(parsed.bucket).file(parsed.objectKey);+ const [metadata] = await raceWithSignal(object.getMetadata(), combined);+ const sizeBytes = Number(metadata.size ?? file.sizeBytes ?? 0);+ // Generations are int64 β keep the SDK's string form, never a JS number.+ const generation =+ metadata.generation !== undefined && metadata.generation !== null+ ? String(metadata.generation)+ : undefined;+ if (!(sizeBytes > 0) || sizeBytes > effectiveMaxBytes) {+ logger.warn("fire-engine GCS file reference has unusable size", {+ uri: file.uri,+ sizeBytes,+ maxBytes: effectiveMaxBytes,+ });+ return null;+ }++ {+ // Pin the read to the generation whose size was just validated, so a+ // concurrent replacement of the object cannot bypass the size gate.+ const pinned =+ generation !== undefined+ ? storage.bucket(parsed.bucket).file(parsed.objectKey, { generation })+ : object;+ await pipeline(pinned.createReadStream(), createWriteStream(destPath), {+ signal: combined,+ });+ }+ return { sizeBytes, generation };+ } catch (error) {+ // A failed stream may have written a partial file that no prefetch+ // cleanup will ever see β remove it here.+ await unlink(destPath).catch(() => {});+ if (signal?.aborted) {+ // Caller cancellation propagates as a cancellation, never as a+ // synthetic prefetch failure.+ throw signal.reason ?? error;+ }+ logger.warn("fire-engine GCS file download failed", {+ uri: file.uri,+ error,+ });+ return null;+ } finally {+ clearTimeout(timer);+ }+}diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.tsindex 749454c1e..b5415a887 100644--- a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts+++ b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts@@ -6,6 +6,7 @@ import os from "os";import { writeFile } from "fs/promises";import { Meta } from "../..";import { documentExtensionFromContentType } from "../../../../lib/document-formats";+import { downloadFireEngineGcsFile } from "./downloadGcsFile";async function feResToFilePrefetch(logger: Logger,@@ -13,8 +14,11 @@ async function feResToFilePrefetch(fileExtension: string,fileType: string,contentType?: string,+ signal?: AbortSignal,+ maxFileBytes?: number,): Promise<Meta["pdfPrefetch"] | Meta["documentPrefetch"]> {- if (!feRes?.file) {+ const file = feRes?.file;+ if (!file || (file.content === undefined && file.gcs_uri === undefined)) {logger.warn(`No file in ${fileType} prefetch`);return null;}@@ -23,7 +27,33 @@ async function feResToFilePrefetch(os.tmpdir(),`tempFile-${crypto.randomUUID()}.${fileExtension}`,);- await writeFile(filePath, Buffer.from(feRes.file.content, "base64"));++ let gcsReference: NonNullable<Meta["pdfPrefetch"]>["gcsReference"];+ if (file.content !== undefined) {+ await writeFile(filePath, Buffer.from(file.content, "base64"));+ } else {+ // Large-file handoff: fire-engine uploaded the bytes to GCS instead of+ // inlining hundreds of MB of base64 through its response and job store.+ // Materialize a local copy (magic-byte sniffing and page-count detection+ // need bytes on disk) and keep the reference so the FirePDF by-reference+ // path can server-side copy the object instead of re-uploading it.+ const downloaded = await downloadFireEngineGcsFile(+ logger,+ { uri: file.gcs_uri!, sha256: file.sha256, sizeBytes: file.size_bytes },+ filePath,+ signal,+ maxFileBytes,+ );+ if (downloaded === null) {+ return null;+ }+ gcsReference = {+ uri: file.gcs_uri!,+ sha256: file.sha256,+ sizeBytes: downloaded.sizeBytes,+ generation: downloaded.generation,+ };+ }return {status: feRes.pageStatusCode,@@ -31,14 +61,27 @@ async function feResToFilePrefetch(filePath,proxyUsed: feRes.usedMobileProxy ? "stealth" : "basic",contentType,+ // References are only produced for PDFs; the document prefetch shape+ // does not carry one.+ ...(fileType === "pdf" && gcsReference ? { gcsReference } : {}),};}async function feResToPdfPrefetch(logger: Logger,feRes: FireEngineCheckStatusSuccess | undefined,+ signal?: AbortSignal,+ maxFileBytes?: number,): Promise<Meta["pdfPrefetch"]> {- return feResToFilePrefetch(logger, feRes, "pdf", "pdf");+ return feResToFilePrefetch(+ logger,+ feRes,+ "pdf",+ "pdf",+ undefined,+ signal,+ maxFileBytes,+ );}async function feResToDocumentPrefetch(@@ -57,11 +100,29 @@ export async function specialtyScrapeCheck(logger: Logger,headers: Record<string, string> | undefined,feRes?: FireEngineCheckStatusSuccess,+ /** Scrape abort signal β stops a large-PDF handoff download when the+ * request has been cancelled or timed out. */+ signal?: AbortSignal,+ /** Admission cap for handoff downloads β the historical download cap when+ * the FirePDF by-reference route is unreachable for this request, so an+ * unusable large handoff never consumes network and temp disk. */+ maxFileBytes?: number,) {const contentType = (Object.entries(headers ?? {}).find(x => x[0].toLowerCase() === "content-type",) ?? [])[1];+ // A GCS reference is a PDF signal on its own β fire-engine only hands+ // off verified PDFs β so it is handled before the content-type guard:+ // some responses omit the header entirely, and the reference must still+ // reach the FirePDF path.+ if (feRes?.file?.gcs_uri !== undefined) {+ throw new AddFeatureError(+ ["pdf"],+ await feResToPdfPrefetch(logger, feRes, signal, maxFileBytes),+ );+ }+if (!contentType) {logger.warn("Failed to check contentType -- was not present in headers", {headers,@@ -89,10 +150,10 @@ export async function specialtyScrapeCheck(// Legacy Office files (.doc, .xls) are OLE2/CFB files starting with D0 CF 11 E0 (base64: "0M8R4K")if (isOctetStream) {const isZipSignature =- feRes?.file?.content.startsWith("UEsD") ||+ feRes?.file?.content?.startsWith("UEsD") ||feRes?.content.startsWith("PK");const isOleSignature =- feRes?.file?.content.startsWith("0M8R4K") ||+ feRes?.file?.content?.startsWith("0M8R4K") ||feRes?.content.startsWith("\xD0\xCF\x11\xE0");if (isZipSignature) {@@ -118,18 +179,24 @@ export async function specialtyScrapeCheck(}}- // Check for PDF+ // Check for PDF (references were already handled above the header guard).if (isPdf) {- throw new AddFeatureError(["pdf"], await feResToPdfPrefetch(logger, feRes));+ throw new AddFeatureError(+ ["pdf"],+ await feResToPdfPrefetch(logger, feRes, signal, maxFileBytes),+ );}// Check for octet-stream with PDF signatureif (isOctetStream &&- (feRes?.file?.content.startsWith("JVBERi0") ||+ (feRes?.file?.content?.startsWith("JVBERi0") ||feRes?.content.startsWith("%PDF-"))) {- throw new AddFeatureError(["pdf"], await feResToPdfPrefetch(logger, feRes));+ throw new AddFeatureError(+ ["pdf"],+ await feResToPdfPrefetch(logger, feRes, signal, maxFileBytes),+ );}// Reject unsupported binary content types (images, video, audio, archives, etc.)diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex c2792ab2c..a1a2ac5eb 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -151,6 +151,19 @@ export type Meta = {status: number;proxyUsed: "basic" | "stealth";contentType?: string;+ /** Set when fire-engine handed the file off by GCS reference (large+ * PDFs): the object it uploaded, so the FirePDF by-reference path+ * can server-side copy it instead of re-uploading the bytes. The+ * local filePath is still materialized (sniffing and page-count+ * detection need bytes on disk). */+ gcsReference?: {+ uri: string;+ sha256?: string;+ sizeBytes?: number;+ /** int64 as the SDK's string form β never rounded through a JS+ * number. */+ generation?: string;+ };}| null| undefined; // undefined: no prefetch yet, null: prefetch came back empty