Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): route mislabeled PDF and Office downloads by their magic bytes (#4502)
Abimael Martellf657a8eaf610
diff --git a/apps/api/src/__tests__/snips/v2/scrape-mislabeled-files.test.ts b/apps/api/src/__tests__/snips/v2/scrape-mislabeled-files.test.tsnew file mode 100644index 000000000..becf8fdeb--- /dev/null+++ b/apps/api/src/__tests__/snips/v2/scrape-mislabeled-files.test.ts@@ -0,0 +1,69 @@+import {+ ALLOW_TEST_SUITE_WEBSITE,+ describeIf,+ TEST_SUITE_WEBSITE,+} from "../lib";+import {+ idmux,+ Identity,+ scrape,+ scrapeTimeout,+ scrapeWithFailure,+} from "./lib";++let identity: Identity;++beforeAll(async () => {+ identity = await idmux({+ name: "scrape-mislabeled-files",+ concurrency: 100,+ credits: 1000000,+ });+}, 10000 + scrapeTimeout);++// The browser engine hands files it cannot render back to the API, which+// routes them to a parser by content type β and servers mislabel. Static+// hosts serve a `.jp2` file as image/jp2 whatever its bytes are, so+// mislabeled-pdf.jp2 reproduces a PDF served with an image content type,+// while tiny-image.jp2 is a real JPEG 2000 image. Only fire-engine performs+// the handoff, hence the gate.+describeIf(!process.env.TEST_SUITE_SELF_HOSTED && ALLOW_TEST_SUITE_WEBSITE)(+ "Mislabeled file handoff (f-e dependent)",+ () => {+ it(+ "parses a PDF served with an image content type",+ async () => {+ const response = await scrape(+ {+ url: `${TEST_SUITE_WEBSITE}/mislabeled-pdf.jp2`,+ formats: ["markdown"],+ },+ identity,+ );++ expect(response.markdown).toContain("Firecrawl mislabeled PDF fixture");+ expect(response.metadata.contentType).toBe("application/pdf");+ expect(response.metadata.numPages).toBe(1);+ expect(response.metadata.statusCode).toBe(200);+ },+ scrapeTimeout,+ );++ it(+ "keeps rejecting a real JPEG 2000 image as unsupported",+ async () => {+ const response = await scrapeWithFailure(+ {+ url: `${TEST_SUITE_WEBSITE}/tiny-image.jp2`,+ formats: ["markdown"],+ },+ identity,+ );++ expect(response.error).toContain("cannot process");+ expect(response.error).toContain("image/jp2");+ },+ scrapeTimeout,+ );+ },+);diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.test.ts b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.test.tsnew file mode 100644index 000000000..7b2dc85c3--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.test.ts@@ -0,0 +1,327 @@+import { afterEach, describe, expect, it, vi } from "vitest";+import { readFile, unlink } from "node:fs/promises";+import { AddFeatureError, UnsupportedFileError } from "../../error";+import type { FireEngineCheckStatusSuccess } from "../fire-engine/checkStatus";++// The GCS handoff (large PDFs by reference) has its own tests under the pdf+// engine; keep the storage client out of this unit's import graph.+vi.mock("./downloadGcsFile", () => ({+ downloadFireEngineGcsFile: vi.fn(async () => null),+}));++import { specialtyScrapeCheck } from "./specialtyHandler";++const logger: any = {+ debug: vi.fn(),+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ child: () => logger,+};++const PDF = Buffer.from(+ "%PDF-1.3\n1 0 obj\n<< /Type /Catalog >>\nendobj\ntrailer\n<< /Root 1 0 R >>\n%%EOF\n",+);+// Leading bytes of an OOXML/ODF (ZIP) container and of a legacy Office+// (OLE2/CFB) file, padded so the payloads are longer than any signature.+const DOCX = Buffer.concat([+ Buffer.from([0x50, 0x4b, 0x03, 0x04, 0x14, 0x00, 0x06, 0x00]),+ Buffer.alloc(64),+]);+const DOC = Buffer.concat([+ Buffer.from([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]),+ Buffer.alloc(64),+]);+const PNG = Buffer.concat([+ Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]),+ Buffer.alloc(64),+]);+// JPEG 2000 signature box: a real image format none of the parsers open.+const JP2 = Buffer.concat([+ Buffer.from([+ 0x00, 0x00, 0x00, 0x0c, 0x6a, 0x50, 0x20, 0x20, 0x0d, 0x0a, 0x87, 0x0a,+ ]),+ Buffer.alloc(64),+]);++const DOWNLOAD_URL = "https://example.com/collection/items/42/download";++/** A chrome-cdp (or tlsclient binary) response: the file was captured. */+function download(+ bytes: Buffer,+ overrides: Partial<FireEngineCheckStatusSuccess> = {},+): FireEngineCheckStatusSuccess {+ return {+ jobId: "job-1",+ state: "completed",+ processing: false,+ content: "",+ pageStatusCode: 200,+ url: DOWNLOAD_URL,+ file: { name: "download", content: bytes.toString("base64") },+ ...overrides,+ };+}++/** A response whose body came back inline as text (no captured file). */+function inline(content: string): FireEngineCheckStatusSuccess {+ return {+ jobId: "job-1",+ state: "completed",+ processing: false,+ content,+ pageStatusCode: 200,+ url: DOWNLOAD_URL,+ };+}++const ocrGate = () => vi.fn(async () => true);++const tempFiles: string[] = [];+afterEach(async () => {+ await Promise.all(+ tempFiles.splice(0).map(file => unlink(file).catch(() => undefined)),+ );+});++async function expectHandoff(run: Promise<unknown>): Promise<AddFeatureError> {+ try {+ await run;+ } catch (error) {+ expect(error).toBeInstanceOf(AddFeatureError);+ const handoff = error as AddFeatureError;+ for (const prefetch of [+ handoff.pdfPrefetch,+ handoff.documentPrefetch,+ handoff.imagePrefetch,+ ]) {+ if (prefetch) tempFiles.push(prefetch.filePath);+ }+ return handoff;+ }+ throw new Error("expected specialtyScrapeCheck to hand the file off");+}++async function expectUnsupported(run: Promise<unknown>): Promise<string> {+ const failure = await run.then(+ () => null,+ (error: unknown) => error,+ );+ expect(failure).toBeInstanceOf(UnsupportedFileError);+ return (failure as UnsupportedFileError).message;+}++describe("specialtyScrapeCheck: the bytes decide when the header does not", () => {+ it.each([+ "image/jp2",+ "image/jpeg",+ "application/x-download",+ "application/force-download",+ "text/plain; charset=utf-8",+ ])("hands a PDF served as %s to the pdf engine", async contentType => {+ const gate = ocrGate();+ const handoff = await expectHandoff(+ specialtyScrapeCheck(+ logger,+ { "Content-Type": contentType },+ download(PDF),+ undefined,+ undefined,+ gate,+ ),+ );++ expect(handoff.featureFlags).toEqual(["pdf"]);+ expect(handoff.pdfPrefetch).toMatchObject({+ status: 200,+ url: DOWNLOAD_URL,+ proxyUsed: "basic",+ });+ expect(handoff.pdfPrefetch!.filePath).toMatch(/\.pdf$/);+ expect(await readFile(handoff.pdfPrefetch!.filePath)).toEqual(PDF);+ expect(handoff.documentPrefetch).toBeUndefined();+ expect(handoff.imagePrefetch).toBeUndefined();+ // Not a raster image, so the per-team OCR gate is never consulted.+ expect(gate).not.toHaveBeenCalled();+ });++ it("carries the proxy that fetched the mislabeled PDF", async () => {+ const handoff = await expectHandoff(+ specialtyScrapeCheck(+ logger,+ { "content-type": "image/jp2" },+ download(PDF, { usedMobileProxy: true }),+ ),+ );+ expect(handoff.pdfPrefetch).toMatchObject({ proxyUsed: "stealth" });+ });++ it("hands an OOXML document served as an image to the document engine", async () => {+ const handoff = await expectHandoff(+ specialtyScrapeCheck(+ logger,+ { "content-type": "image/png" },+ download(DOCX),+ undefined,+ undefined,+ ocrGate(),+ ),+ );++ expect(handoff.featureFlags).toEqual(["document"]);+ // The header is passed through as served: the document engine treats it+ // as an extension hint only and falls back to the URL and the bytes.+ expect(handoff.documentPrefetch).toMatchObject({+ status: 200,+ url: DOWNLOAD_URL,+ contentType: "image/png",+ });+ expect(await readFile(handoff.documentPrefetch!.filePath)).toEqual(DOCX);+ expect(handoff.pdfPrefetch).toBeUndefined();+ expect(handoff.imagePrefetch).toBeUndefined();+ });++ it("hands a legacy Office file to the document engine, pinning .doc URLs to Word", async () => {+ const generic = await expectHandoff(+ specialtyScrapeCheck(+ logger,+ { "content-type": "image/jp2" },+ download(DOC),+ ),+ );+ expect(generic.featureFlags).toEqual(["document"]);+ expect(generic.documentPrefetch?.contentType).toBe("image/jp2");++ const word = await expectHandoff(+ specialtyScrapeCheck(+ logger,+ { "content-type": "image/jp2" },+ download(DOC, { url: "https://example.com/files/report.doc?dl=1" }),+ ),+ );+ expect(word.featureFlags).toEqual(["document"]);+ expect(word.documentPrefetch?.contentType).toBe("application/msword");+ expect(word.documentPrefetch?.filePath).toMatch(/\.doc$/);+ });++ it("still sends a real raster image to OCR before sniffing for documents", async () => {+ const gate = ocrGate();+ const handoff = await expectHandoff(+ specialtyScrapeCheck(+ logger,+ { "content-type": "image/jp2" },+ download(PNG),+ undefined,+ undefined,+ gate,+ ),+ );++ expect(handoff.featureFlags).toEqual(["image"]);+ expect(handoff.imagePrefetch).toMatchObject({ contentType: "image/png" });+ expect(handoff.pdfPrefetch).toBeUndefined();+ expect(gate).toHaveBeenCalledTimes(1);+ });++ it("keeps rejecting an image format no parser can open", async () => {+ const gate = ocrGate();+ const message = await expectUnsupported(+ specialtyScrapeCheck(+ logger,+ { "content-type": "image/jp2" },+ download(JP2),+ undefined,+ undefined,+ gate,+ ),+ );+ expect(message).toContain("image/jp2");+ expect(gate).not.toHaveBeenCalled();+ });++ it("leaves a declared ZIP archive with the unsupported-file rejection", async () => {+ const message = await expectUnsupported(+ specialtyScrapeCheck(+ logger,+ { "content-type": "application/zip" },+ download(DOCX),+ ),+ );+ expect(message).toContain("application/zip");+ });++ it("keeps routing application/octet-stream by its bytes", async () => {+ const pdf = await expectHandoff(+ specialtyScrapeCheck(+ logger,+ { "content-type": "application/octet-stream" },+ download(PDF),+ ),+ );+ expect(pdf.featureFlags).toEqual(["pdf"]);++ const docx = await expectHandoff(+ specialtyScrapeCheck(+ logger,+ { "content-type": "application/octet-stream" },+ download(DOCX),+ ),+ );+ expect(docx.featureFlags).toEqual(["document"]);+ expect(docx.documentPrefetch?.contentType).toBe("application/octet-stream");+ });++ it("sniffs a body fire-engine returned inline as text", async () => {+ // tlsclient returns UTF-8 bodies as `content` rather than as a file. A+ // body that opens with the PDF magic is still a PDF; with nothing+ // captured the prefetch is null, so the pdf engine fetches it itself.+ const handoff = await expectHandoff(+ specialtyScrapeCheck(+ logger,+ { "content-type": "application/x-download" },+ inline("%PDF-1.4\n1 0 obj\n<< /Type /Catalog >>\nendobj\n"),+ ),+ );+ expect(handoff.featureFlags).toEqual(["pdf"]);+ expect(handoff.pdfPrefetch).toBeNull();+ });++ it("does not sniff an ordinary rendered page", async () => {+ await expect(+ specialtyScrapeCheck(+ logger,+ { "content-type": "text/html; charset=utf-8" },+ inline("<!DOCTYPE html><html><body>PK</body></html>"),+ ),+ ).resolves.toBeUndefined();+ });++ it("does not mistake a text body that merely starts with PK for an archive", async () => {+ await expect(+ specialtyScrapeCheck(+ logger,+ { "content-type": "text/plain" },+ inline("PKI certificate bundle\n"),+ ),+ ).resolves.toBeUndefined();+ });++ it("recognizes a header-less download by its bytes", async () => {+ const pdf = await expectHandoff(+ specialtyScrapeCheck(logger, undefined, download(PDF)),+ );+ expect(pdf.featureFlags).toEqual(["pdf"]);++ const docx = await expectHandoff(+ specialtyScrapeCheck(logger, {}, download(DOCX)),+ );+ expect(docx.featureFlags).toEqual(["document"]);+ expect(docx.documentPrefetch?.contentType).toBeUndefined();+ });++ it("does nothing without a fire-engine response to sniff", async () => {+ await expect(+ specialtyScrapeCheck(logger, { "content-type": "text/html" }),+ ).resolves.toBeUndefined();+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.tsindex 442d972d4..586a55b17 100644--- a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts+++ b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts@@ -94,7 +94,7 @@ async function feResToPdfPrefetch(async function feResToDocumentPrefetch(logger: Logger,feRes: FireEngineCheckStatusSuccess | undefined,- contentType: string,+ contentType: string | undefined,): Promise<Meta["documentPrefetch"]> {// Determine file extension from content typeconst extension =@@ -134,6 +134,119 @@ async function sniffImageHandoff(return (await imageOcrEnabled()) ? contentType : null;}+/**+ * File types the parser engines can open, recognized from magic bytes.+ * Servers mislabel freely β a PDF behind a generic download endpoint comes+ * back as application/octet-stream or image/jp2, an Office file as+ * application/x-download β and a browser can only hand off what it could not+ * render, so once the header has failed to name a parseable type the bytes+ * decide which engine gets the file.+ */+type HandoffFileKind = "pdf" | "zip" | "ole";++const HANDOFF_FILE_SIGNATURES: Array<{+ kind: HandoffFileKind;+ bytes: number[];+}> = [+ // "%PDF-"+ { kind: "pdf", bytes: [0x25, 0x50, 0x44, 0x46, 0x2d] },+ // "PK\x03\x04": the local file header every OOXML/ODF container (.docx,+ // .xlsx, .pptx, .odt, ...) starts with. All four bytes, so a text body that+ // merely begins with "PK" is not mistaken for an archive.+ { kind: "zip", bytes: [0x50, 0x4b, 0x03, 0x04] },+ // OLE2/CFB header shared by legacy Office files (.doc, .xls, .ppt).+ { kind: "ole", bytes: [0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1] },+];++const HANDOFF_SNIFF_BYTES = Math.max(+ ...HANDOFF_FILE_SIGNATURES.map(s => s.bytes.length),+);++// Headers that already declare a ZIP archive. A ZIP signature under one of+// these confirms a real archive rather than exposing a mislabeled Office+// file, so it keeps the unsupported-file rejection instead of being sent to+// the document parser.+const ZIP_ARCHIVE_MEDIA_TYPES = new Set([+ "application/zip",+ "application/x-zip-compressed",+ "application/java-archive",+]);++function matchHandoffSignature(+ head: ArrayLike<number>,+): HandoffFileKind | null {+ for (const { kind, bytes } of HANDOFF_FILE_SIGNATURES) {+ if (head.length < bytes.length) continue;+ let matches = true;+ for (let i = 0; i < bytes.length; i++) {+ if (head[i] !== bytes[i]) {+ matches = false;+ break;+ }+ }+ if (matches) return kind;+ }+ return null;+}++/**+ * Identifies a handoff's file type from its leading bytes. A captured+ * download arrives base64-encoded in `file.content` (4 characters encode 3+ * bytes, so a short prefix covers the longest signature without decoding the+ * file); a body fire-engine returned as text sits in `content` and is+ * compared character by character.+ */+function sniffHandoffFileKind(+ feRes: FireEngineCheckStatusSuccess | undefined,+): HandoffFileKind | null {+ if (!feRes) return null;+ if (feRes.file?.content !== undefined) {+ const chars = Math.ceil(HANDOFF_SNIFF_BYTES / 3) * 4;+ const kind = matchHandoffSignature(+ Buffer.from(feRes.file.content.slice(0, chars), "base64"),+ );+ if (kind !== null) return kind;+ }+ return matchHandoffSignature(+ Array.from(feRes.content.slice(0, HANDOFF_SNIFF_BYTES), c =>+ c.charCodeAt(0),+ ),+ );+}++/**+ * Hands a sniffed file to the engine that parses it. `contentType` is the+ * header as served (undefined without one); the document engine only uses it+ * as an extension hint before falling back to the URL path and the+ * converter's own detection.+ */+async function handoffSniffedFile(+ kind: HandoffFileKind,+ logger: Logger,+ feRes: FireEngineCheckStatusSuccess | undefined,+ contentType: string | undefined,+ signal?: AbortSignal,+ maxFileBytes?: number,+): Promise<never> {+ if (kind === "pdf") {+ throw new AddFeatureError(+ ["pdf"],+ await feResToPdfPrefetch(logger, feRes, signal, maxFileBytes),+ );+ }+ // OLE2 signature is shared by .doc/.xls/.ppt files+ // Only override to application/msword if URL suggests it's a .doc file+ const url = feRes?.url?.toLowerCase() ?? "";+ const isDocUrl = url.endsWith(".doc") || url.includes(".doc?");+ const effectiveContentType =+ kind === "ole" && isDocUrl ? "application/msword" : contentType;+ throw new AddFeatureError(+ ["document"],+ undefined,+ await feResToDocumentPrefetch(logger, feRes, effectiveContentType),+ );+}+export async function specialtyScrapeCheck(logger: Logger,headers: Record<string, string> | undefined,@@ -176,6 +289,17 @@ export async function specialtyScrapeCheck(await feResToImagePrefetch(logger, feRes, headerlessImage),);}+ const headerlessFile = sniffHandoffFileKind(feRes);+ if (headerlessFile !== null) {+ await handoffSniffedFile(+ headerlessFile,+ logger,+ feRes,+ undefined,+ signal,+ maxFileBytes,+ );+ }logger.warn("Failed to check contentType -- was not present in headers", {headers,});@@ -186,7 +310,6 @@ export async function specialtyScrapeCheck(const isPdf =contentType === "application/pdf" ||contentType.startsWith("application/pdf;");- const isOctetStream = contentType === "application/octet-stream";// Check for document types first (before PDF to prioritize documents)if (isDocument) {@@ -197,40 +320,6 @@ export async function specialtyScrapeCheck();}- // Check for octet-stream with document signature- // Modern Office files (.docx, .xlsx) are ZIP archives starting with "PK" (base64: "UEsD")- // Legacy Office files (.doc, .xls) are OLE2/CFB files starting with D0 CF 11 E0 (base64: "0M8R4K")- if (isOctetStream) {- const isZipSignature =- feRes?.file?.content?.startsWith("UEsD") ||- feRes?.content.startsWith("PK");- const isOleSignature =- feRes?.file?.content?.startsWith("0M8R4K") ||- feRes?.content.startsWith("\xD0\xCF\x11\xE0");-- if (isZipSignature) {- throw new AddFeatureError(- ["document"],- undefined,- await feResToDocumentPrefetch(logger, feRes, contentType),- );- }- if (isOleSignature) {- // OLE2 signature is shared by .doc/.xls/.ppt files- // Only override to application/msword if URL suggests it's a .doc file- const url = feRes?.url?.toLowerCase() ?? "";- const isDocUrl = url.endsWith(".doc") || url.includes(".doc?");- const effectiveContentType = isDocUrl- ? "application/msword"- : contentType;- throw new AddFeatureError(- ["document"],- undefined,- await feResToDocumentPrefetch(logger, feRes, effectiveContentType),- );- }- }-// Check for PDF (references were already handled above the header guard).if (isPdf) {throw new AddFeatureError(@@ -239,24 +328,13 @@ export async function specialtyScrapeCheck();}- // Check for octet-stream with PDF signature- if (- isOctetStream &&- (feRes?.file?.content?.startsWith("JVBERi0") ||- feRes?.content.startsWith("%PDF-"))- ) {- throw new AddFeatureError(- ["pdf"],- await feResToPdfPrefetch(logger, feRes, signal, maxFileBytes),- );- }+ const mediaType = contentType.split(";")[0].trim().toLowerCase();// Raster images are OCR'd as one-page scanned documents by the image// engine (see engines/image). The header must claim an image or an opaque// octet-stream AND the bytes must confirm a format FirePDF can open, so a// mislabeled HTML error page served as image/jpeg is never sent to OCR// and WebP/SVG/AVIF keep failing fast below.- const mediaType = contentType.split(";")[0].trim().toLowerCase();if (mediaType.startsWith("image/") ||mediaType === "application/octet-stream"@@ -272,6 +350,27 @@ export async function specialtyScrapeCheck(}}+ // The header named no parseable type, but the bytes may still: a PDF served+ // as application/octet-stream or image/jp2, an Office file behind a generic+ // download endpoint. The browser only hands off what it could not render,+ // so every remaining content type is sniffed β after the image check above,+ // so a real raster image still reaches OCR, and before the rejection below,+ // so a mislabeled PDF is parsed instead of failing as an unsupported image.+ const sniffedFile = sniffHandoffFileKind(feRes);+ if (+ sniffedFile !== null &&+ !(sniffedFile === "zip" && ZIP_ARCHIVE_MEDIA_TYPES.has(mediaType))+ ) {+ await handoffSniffedFile(+ sniffedFile,+ logger,+ feRes,+ contentType,+ signal,+ maxFileBytes,+ );+ }+// Reject unsupported binary content types (unsupported images, video, audio, archives, etc.)const unsupportedBinaryPrefixes = ["image/",diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 2857705a9..b1e8b3138 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -24,6 +24,7 @@ import {scrapeURLWithEngine,shouldUseIndex,} from "./engines";+import { applyHandoffFeatureFlags } from "./lib/handoffFeatureFlags";import { parseMarkdown } from "../../lib/html-to-markdown";import { hasFormatOfType } from "../../lib/format-utils";import {@@ -1386,14 +1387,24 @@ export async function scrapeURL(Array.isArray(meta.internalOptions.forceEngine))) {retryTracker.record("feature_toggle", error);+ // A file handoff names the one parser that can open the file;+ // the file flag the URL extension implied earlier gives way to it.+ const nextFeatureFlags = applyHandoffFeatureFlags(+ meta.featureFlags,+ error.featureFlags,+ );meta.logger.debug("More feature flags requested by scraper: adding " +error.featureFlags.join(", "),- { error, existingFlags: meta.featureFlags },- );- meta.featureFlags = new Set(- [...meta.featureFlags].concat(error.featureFlags),+ {+ error,+ existingFlags: meta.featureFlags,+ droppedFlags: [...meta.featureFlags].filter(+ flag => !nextFeatureFlags.has(flag),+ ),+ },);+ meta.featureFlags = nextFeatureFlags;if (error.pdfPrefetch) {meta.pdfPrefetch = error.pdfPrefetch;} else if (error.pdfPrefetch === null) {diff --git a/apps/api/src/scraper/scrapeURL/lib/handoffFeatureFlags.test.ts b/apps/api/src/scraper/scrapeURL/lib/handoffFeatureFlags.test.tsnew file mode 100644index 000000000..f2a0be241--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/lib/handoffFeatureFlags.test.ts@@ -0,0 +1,46 @@+import { describe, expect, it } from "vitest";+import type { FeatureFlag } from "../engines";+import { applyHandoffFeatureFlags } from "./handoffFeatureFlags";++describe("applyHandoffFeatureFlags", () => {+ it("replaces the URL-derived file flag with the parser the handoff names", () => {+ // A .pdf URL that served a docx.+ expect(applyHandoffFeatureFlags(new Set(["pdf"]), ["document"])).toEqual(+ new Set(["document"]),+ );+ // A .docx URL that served a PDF.+ expect(applyHandoffFeatureFlags(new Set(["document"]), ["pdf"])).toEqual(+ new Set(["pdf"]),+ );+ // A .pdf URL that served a PNG.+ expect(applyHandoffFeatureFlags(new Set(["pdf"]), ["image"])).toEqual(+ new Set(["image"]),+ );+ });++ it("keeps every non-file flag", () => {+ expect(+ applyHandoffFeatureFlags(new Set(["pdf", "location", "waitFor"]), [+ "document",+ ]),+ ).toEqual(new Set(["document", "location", "waitFor"]));+ });++ it("is a plain union when the handoff names no parser", () => {+ expect(+ applyHandoffFeatureFlags(new Set(["pdf", "location"]), ["stealthProxy"]),+ ).toEqual(new Set(["pdf", "location", "stealthProxy"]));+ });++ it("leaves a matching file flag in place", () => {+ expect(+ applyHandoffFeatureFlags(new Set(["pdf", "mobile"]), ["pdf"]),+ ).toEqual(new Set(["pdf", "mobile"]));+ });++ it("does not mutate the request's set", () => {+ const current = new Set<FeatureFlag>(["pdf"]);+ applyHandoffFeatureFlags(current, ["document"]);+ expect(current).toEqual(new Set(["pdf"]));+ });+});diff --git a/apps/api/src/scraper/scrapeURL/lib/handoffFeatureFlags.ts b/apps/api/src/scraper/scrapeURL/lib/handoffFeatureFlags.tsnew file mode 100644index 000000000..ed5026058--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/lib/handoffFeatureFlags.ts@@ -0,0 +1,44 @@+import type { FeatureFlag } from "../engines";++/**+ * The parser flags a file handoff can name. A file is exactly one of these:+ * the bytes (or header) that produced the handoff decide which.+ */+const FILE_FEATURE_FLAGS: ReadonlySet<FeatureFlag> = new Set<FeatureFlag>([+ "pdf",+ "document",+ "image",+]);++/**+ * Merges a handoff's feature flags into the request's set.+ *+ * A file flag the URL extension implied up front (the `pdf` of a .pdf URL+ * that turns out to serve a docx) is dropped when the handoff names a+ * different parser. The file flags weigh 100 each in buildFallbackList's+ * support threshold, so keeping both would demand a parser that handles both+ * types: with nothing else set the single-type parsers only just qualify, and+ * any further flag (location, stealthProxy, waitFor) pushes the threshold+ * past them, stranding a file that is already in hand.+ */+export function applyHandoffFeatureFlags(+ current: ReadonlySet<FeatureFlag>,+ added: readonly FeatureFlag[],+): Set<FeatureFlag> {+ const handedOff = added.filter(flag => FILE_FEATURE_FLAGS.has(flag));+ const next = new Set<FeatureFlag>();+ for (const flag of current) {+ if (+ handedOff.length > 0 &&+ FILE_FEATURE_FLAGS.has(flag) &&+ !handedOff.includes(flag)+ ) {+ continue;+ }+ next.add(flag);+ }+ for (const flag of added) {+ next.add(flag);+ }+ return next;+}diff --git a/apps/test-site/public/mislabeled-pdf.jp2 b/apps/test-site/public/mislabeled-pdf.jp2new file mode 100644index 000000000..186d7ae47--- /dev/null+++ b/apps/test-site/public/mislabeled-pdf.jp2@@ -0,0 +1,77 @@+%PDF-1.3+1 0 obj+<< /Type /Catalog /Pages 2 0 R >>+endobj+2 0 obj+<< /Type /Pages /Kids [3 0 R] /Count 1 >>+endobj+3 0 obj+<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>+endobj+4 0 obj+<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>+endobj+5 0 obj+<< /Length 5170 >>+stream+BT /F1 18 Tf 72 730 Td (Firecrawl mislabeled PDF fixture) Tj ET+BT /F1 12 Tf 72 706 Td (Served with an image content type, but the bytes are a PDF.) Tj ET+BT /F1 9 Tf 72 680 Td (Fixture body line 01 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 667 Td (Fixture body line 02 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 654 Td (Fixture body line 03 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 641 Td (Fixture body line 04 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 628 Td (Fixture body line 05 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 615 Td (Fixture body line 06 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 602 Td (Fixture body line 07 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 589 Td (Fixture body line 08 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 576 Td (Fixture body line 09 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 563 Td (Fixture body line 10 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 550 Td (Fixture body line 11 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 537 Td (Fixture body line 12 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 524 Td (Fixture body line 13 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 511 Td (Fixture body line 14 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 498 Td (Fixture body line 15 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 485 Td (Fixture body line 16 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 472 Td (Fixture body line 17 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 459 Td (Fixture body line 18 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 446 Td (Fixture body line 19 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 433 Td (Fixture body line 20 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 420 Td (Fixture body line 21 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 407 Td (Fixture body line 22 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 394 Td (Fixture body line 23 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 381 Td (Fixture body line 24 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 368 Td (Fixture body line 25 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 355 Td (Fixture body line 26 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 342 Td (Fixture body line 27 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 329 Td (Fixture body line 28 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 316 Td (Fixture body line 29 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 303 Td (Fixture body line 30 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 290 Td (Fixture body line 31 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 277 Td (Fixture body line 32 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 264 Td (Fixture body line 33 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 251 Td (Fixture body line 34 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 238 Td (Fixture body line 35 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 225 Td (Fixture body line 36 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 212 Td (Fixture body line 37 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 199 Td (Fixture body line 38 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 186 Td (Fixture body line 39 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 173 Td (Fixture body line 40 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 160 Td (Fixture body line 41 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 147 Td (Fixture body line 42 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 134 Td (Fixture body line 43 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+BT /F1 9 Tf 72 121 Td (Fixture body line 44 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET+endstream+endobj+xref+0 6+0000000000 65535 f+0000000009 00000 n+0000000058 00000 n+0000000115 00000 n+0000000241 00000 n+0000000311 00000 n+trailer+<< /Size 6 /Root 1 0 R >>+startxref+5533+%%EOFdiff --git a/apps/test-site/public/tiny-image.jp2 b/apps/test-site/public/tiny-image.jp2new file mode 100644index 000000000..90b925bbcBinary files /dev/null and b/apps/test-site/public/tiny-image.jp2 differ