Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

fix(api): route mislabeled PDF and Office downloads by their magic bytes (#4502)

Abimael Martellf657a8eaf610
diff --git a/apps/api/src/__tests__/snips/v2/scrape-mislabeled-files.test.ts b/apps/api/src/__tests__/snips/v2/scrape-mislabeled-files.test.ts
new file mode 100644
index 000000000..becf8fdeb
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v2/scrape-mislabeled-files.test.ts
@@ -0,0 +1,69 @@
+import {
+ ALLOW_TEST_SUITE_WEBSITE,
+ describeIf,
+ TEST_SUITE_WEBSITE,
+} from "../lib";
+import {
+ idmux,
+ Identity,
+ scrape,
+ scrapeTimeout,
+ scrapeWithFailure,
+} from "./lib";
+
+let identity: Identity;
+
+beforeAll(async () => {
+ identity = await idmux({
+ name: "scrape-mislabeled-files",
+ concurrency: 100,
+ credits: 1000000,
+ });
+}, 10000 + scrapeTimeout);
+
+// The browser engine hands files it cannot render back to the API, which
+// routes them to a parser by content type β€” and servers mislabel. Static
+// hosts serve a `.jp2` file as image/jp2 whatever its bytes are, so
+// mislabeled-pdf.jp2 reproduces a PDF served with an image content type,
+// while tiny-image.jp2 is a real JPEG 2000 image. Only fire-engine performs
+// the handoff, hence the gate.
+describeIf(!process.env.TEST_SUITE_SELF_HOSTED && ALLOW_TEST_SUITE_WEBSITE)(
+ "Mislabeled file handoff (f-e dependent)",
+ () => {
+ it(
+ "parses a PDF served with an image content type",
+ async () => {
+ const response = await scrape(
+ {
+ url: `${TEST_SUITE_WEBSITE}/mislabeled-pdf.jp2`,
+ formats: ["markdown"],
+ },
+ identity,
+ );
+
+ expect(response.markdown).toContain("Firecrawl mislabeled PDF fixture");
+ expect(response.metadata.contentType).toBe("application/pdf");
+ expect(response.metadata.numPages).toBe(1);
+ expect(response.metadata.statusCode).toBe(200);
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "keeps rejecting a real JPEG 2000 image as unsupported",
+ async () => {
+ const response = await scrapeWithFailure(
+ {
+ url: `${TEST_SUITE_WEBSITE}/tiny-image.jp2`,
+ formats: ["markdown"],
+ },
+ identity,
+ );
+
+ expect(response.error).toContain("cannot process");
+ expect(response.error).toContain("image/jp2");
+ },
+ scrapeTimeout,
+ );
+ },
+);
diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.test.ts b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.test.ts
new file mode 100644
index 000000000..7b2dc85c3
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.test.ts
@@ -0,0 +1,327 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { readFile, unlink } from "node:fs/promises";
+import { AddFeatureError, UnsupportedFileError } from "../../error";
+import type { FireEngineCheckStatusSuccess } from "../fire-engine/checkStatus";
+
+// The GCS handoff (large PDFs by reference) has its own tests under the pdf
+// engine; keep the storage client out of this unit's import graph.
+vi.mock("./downloadGcsFile", () => ({
+ downloadFireEngineGcsFile: vi.fn(async () => null),
+}));
+
+import { specialtyScrapeCheck } from "./specialtyHandler";
+
+const logger: any = {
+ debug: vi.fn(),
+ info: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ child: () => logger,
+};
+
+const PDF = Buffer.from(
+ "%PDF-1.3\n1 0 obj\n<< /Type /Catalog >>\nendobj\ntrailer\n<< /Root 1 0 R >>\n%%EOF\n",
+);
+// Leading bytes of an OOXML/ODF (ZIP) container and of a legacy Office
+// (OLE2/CFB) file, padded so the payloads are longer than any signature.
+const DOCX = Buffer.concat([
+ Buffer.from([0x50, 0x4b, 0x03, 0x04, 0x14, 0x00, 0x06, 0x00]),
+ Buffer.alloc(64),
+]);
+const DOC = Buffer.concat([
+ Buffer.from([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]),
+ Buffer.alloc(64),
+]);
+const PNG = Buffer.concat([
+ Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]),
+ Buffer.alloc(64),
+]);
+// JPEG 2000 signature box: a real image format none of the parsers open.
+const JP2 = Buffer.concat([
+ Buffer.from([
+ 0x00, 0x00, 0x00, 0x0c, 0x6a, 0x50, 0x20, 0x20, 0x0d, 0x0a, 0x87, 0x0a,
+ ]),
+ Buffer.alloc(64),
+]);
+
+const DOWNLOAD_URL = "https://example.com/collection/items/42/download";
+
+/** A chrome-cdp (or tlsclient binary) response: the file was captured. */
+function download(
+ bytes: Buffer,
+ overrides: Partial<FireEngineCheckStatusSuccess> = {},
+): FireEngineCheckStatusSuccess {
+ return {
+ jobId: "job-1",
+ state: "completed",
+ processing: false,
+ content: "",
+ pageStatusCode: 200,
+ url: DOWNLOAD_URL,
+ file: { name: "download", content: bytes.toString("base64") },
+ ...overrides,
+ };
+}
+
+/** A response whose body came back inline as text (no captured file). */
+function inline(content: string): FireEngineCheckStatusSuccess {
+ return {
+ jobId: "job-1",
+ state: "completed",
+ processing: false,
+ content,
+ pageStatusCode: 200,
+ url: DOWNLOAD_URL,
+ };
+}
+
+const ocrGate = () => vi.fn(async () => true);
+
+const tempFiles: string[] = [];
+afterEach(async () => {
+ await Promise.all(
+ tempFiles.splice(0).map(file => unlink(file).catch(() => undefined)),
+ );
+});
+
+async function expectHandoff(run: Promise<unknown>): Promise<AddFeatureError> {
+ try {
+ await run;
+ } catch (error) {
+ expect(error).toBeInstanceOf(AddFeatureError);
+ const handoff = error as AddFeatureError;
+ for (const prefetch of [
+ handoff.pdfPrefetch,
+ handoff.documentPrefetch,
+ handoff.imagePrefetch,
+ ]) {
+ if (prefetch) tempFiles.push(prefetch.filePath);
+ }
+ return handoff;
+ }
+ throw new Error("expected specialtyScrapeCheck to hand the file off");
+}
+
+async function expectUnsupported(run: Promise<unknown>): Promise<string> {
+ const failure = await run.then(
+ () => null,
+ (error: unknown) => error,
+ );
+ expect(failure).toBeInstanceOf(UnsupportedFileError);
+ return (failure as UnsupportedFileError).message;
+}
+
+describe("specialtyScrapeCheck: the bytes decide when the header does not", () => {
+ it.each([
+ "image/jp2",
+ "image/jpeg",
+ "application/x-download",
+ "application/force-download",
+ "text/plain; charset=utf-8",
+ ])("hands a PDF served as %s to the pdf engine", async contentType => {
+ const gate = ocrGate();
+ const handoff = await expectHandoff(
+ specialtyScrapeCheck(
+ logger,
+ { "Content-Type": contentType },
+ download(PDF),
+ undefined,
+ undefined,
+ gate,
+ ),
+ );
+
+ expect(handoff.featureFlags).toEqual(["pdf"]);
+ expect(handoff.pdfPrefetch).toMatchObject({
+ status: 200,
+ url: DOWNLOAD_URL,
+ proxyUsed: "basic",
+ });
+ expect(handoff.pdfPrefetch!.filePath).toMatch(/\.pdf$/);
+ expect(await readFile(handoff.pdfPrefetch!.filePath)).toEqual(PDF);
+ expect(handoff.documentPrefetch).toBeUndefined();
+ expect(handoff.imagePrefetch).toBeUndefined();
+ // Not a raster image, so the per-team OCR gate is never consulted.
+ expect(gate).not.toHaveBeenCalled();
+ });
+
+ it("carries the proxy that fetched the mislabeled PDF", async () => {
+ const handoff = await expectHandoff(
+ specialtyScrapeCheck(
+ logger,
+ { "content-type": "image/jp2" },
+ download(PDF, { usedMobileProxy: true }),
+ ),
+ );
+ expect(handoff.pdfPrefetch).toMatchObject({ proxyUsed: "stealth" });
+ });
+
+ it("hands an OOXML document served as an image to the document engine", async () => {
+ const handoff = await expectHandoff(
+ specialtyScrapeCheck(
+ logger,
+ { "content-type": "image/png" },
+ download(DOCX),
+ undefined,
+ undefined,
+ ocrGate(),
+ ),
+ );
+
+ expect(handoff.featureFlags).toEqual(["document"]);
+ // The header is passed through as served: the document engine treats it
+ // as an extension hint only and falls back to the URL and the bytes.
+ expect(handoff.documentPrefetch).toMatchObject({
+ status: 200,
+ url: DOWNLOAD_URL,
+ contentType: "image/png",
+ });
+ expect(await readFile(handoff.documentPrefetch!.filePath)).toEqual(DOCX);
+ expect(handoff.pdfPrefetch).toBeUndefined();
+ expect(handoff.imagePrefetch).toBeUndefined();
+ });
+
+ it("hands a legacy Office file to the document engine, pinning .doc URLs to Word", async () => {
+ const generic = await expectHandoff(
+ specialtyScrapeCheck(
+ logger,
+ { "content-type": "image/jp2" },
+ download(DOC),
+ ),
+ );
+ expect(generic.featureFlags).toEqual(["document"]);
+ expect(generic.documentPrefetch?.contentType).toBe("image/jp2");
+
+ const word = await expectHandoff(
+ specialtyScrapeCheck(
+ logger,
+ { "content-type": "image/jp2" },
+ download(DOC, { url: "https://example.com/files/report.doc?dl=1" }),
+ ),
+ );
+ expect(word.featureFlags).toEqual(["document"]);
+ expect(word.documentPrefetch?.contentType).toBe("application/msword");
+ expect(word.documentPrefetch?.filePath).toMatch(/\.doc$/);
+ });
+
+ it("still sends a real raster image to OCR before sniffing for documents", async () => {
+ const gate = ocrGate();
+ const handoff = await expectHandoff(
+ specialtyScrapeCheck(
+ logger,
+ { "content-type": "image/jp2" },
+ download(PNG),
+ undefined,
+ undefined,
+ gate,
+ ),
+ );
+
+ expect(handoff.featureFlags).toEqual(["image"]);
+ expect(handoff.imagePrefetch).toMatchObject({ contentType: "image/png" });
+ expect(handoff.pdfPrefetch).toBeUndefined();
+ expect(gate).toHaveBeenCalledTimes(1);
+ });
+
+ it("keeps rejecting an image format no parser can open", async () => {
+ const gate = ocrGate();
+ const message = await expectUnsupported(
+ specialtyScrapeCheck(
+ logger,
+ { "content-type": "image/jp2" },
+ download(JP2),
+ undefined,
+ undefined,
+ gate,
+ ),
+ );
+ expect(message).toContain("image/jp2");
+ expect(gate).not.toHaveBeenCalled();
+ });
+
+ it("leaves a declared ZIP archive with the unsupported-file rejection", async () => {
+ const message = await expectUnsupported(
+ specialtyScrapeCheck(
+ logger,
+ { "content-type": "application/zip" },
+ download(DOCX),
+ ),
+ );
+ expect(message).toContain("application/zip");
+ });
+
+ it("keeps routing application/octet-stream by its bytes", async () => {
+ const pdf = await expectHandoff(
+ specialtyScrapeCheck(
+ logger,
+ { "content-type": "application/octet-stream" },
+ download(PDF),
+ ),
+ );
+ expect(pdf.featureFlags).toEqual(["pdf"]);
+
+ const docx = await expectHandoff(
+ specialtyScrapeCheck(
+ logger,
+ { "content-type": "application/octet-stream" },
+ download(DOCX),
+ ),
+ );
+ expect(docx.featureFlags).toEqual(["document"]);
+ expect(docx.documentPrefetch?.contentType).toBe("application/octet-stream");
+ });
+
+ it("sniffs a body fire-engine returned inline as text", async () => {
+ // tlsclient returns UTF-8 bodies as `content` rather than as a file. A
+ // body that opens with the PDF magic is still a PDF; with nothing
+ // captured the prefetch is null, so the pdf engine fetches it itself.
+ const handoff = await expectHandoff(
+ specialtyScrapeCheck(
+ logger,
+ { "content-type": "application/x-download" },
+ inline("%PDF-1.4\n1 0 obj\n<< /Type /Catalog >>\nendobj\n"),
+ ),
+ );
+ expect(handoff.featureFlags).toEqual(["pdf"]);
+ expect(handoff.pdfPrefetch).toBeNull();
+ });
+
+ it("does not sniff an ordinary rendered page", async () => {
+ await expect(
+ specialtyScrapeCheck(
+ logger,
+ { "content-type": "text/html; charset=utf-8" },
+ inline("<!DOCTYPE html><html><body>PK</body></html>"),
+ ),
+ ).resolves.toBeUndefined();
+ });
+
+ it("does not mistake a text body that merely starts with PK for an archive", async () => {
+ await expect(
+ specialtyScrapeCheck(
+ logger,
+ { "content-type": "text/plain" },
+ inline("PKI certificate bundle\n"),
+ ),
+ ).resolves.toBeUndefined();
+ });
+
+ it("recognizes a header-less download by its bytes", async () => {
+ const pdf = await expectHandoff(
+ specialtyScrapeCheck(logger, undefined, download(PDF)),
+ );
+ expect(pdf.featureFlags).toEqual(["pdf"]);
+
+ const docx = await expectHandoff(
+ specialtyScrapeCheck(logger, {}, download(DOCX)),
+ );
+ expect(docx.featureFlags).toEqual(["document"]);
+ expect(docx.documentPrefetch?.contentType).toBeUndefined();
+ });
+
+ it("does nothing without a fire-engine response to sniff", async () => {
+ await expect(
+ specialtyScrapeCheck(logger, { "content-type": "text/html" }),
+ ).resolves.toBeUndefined();
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
index 442d972d4..586a55b17 100644
--- a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
@@ -94,7 +94,7 @@ async function feResToPdfPrefetch(
async function feResToDocumentPrefetch(
logger: Logger,
feRes: FireEngineCheckStatusSuccess | undefined,
- contentType: string,
+ contentType: string | undefined,
): Promise<Meta["documentPrefetch"]> {
// Determine file extension from content type
const extension =
@@ -134,6 +134,119 @@ async function sniffImageHandoff(
return (await imageOcrEnabled()) ? contentType : null;
}
+/**
+ * File types the parser engines can open, recognized from magic bytes.
+ * Servers mislabel freely β€” a PDF behind a generic download endpoint comes
+ * back as application/octet-stream or image/jp2, an Office file as
+ * application/x-download β€” and a browser can only hand off what it could not
+ * render, so once the header has failed to name a parseable type the bytes
+ * decide which engine gets the file.
+ */
+type HandoffFileKind = "pdf" | "zip" | "ole";
+
+const HANDOFF_FILE_SIGNATURES: Array<{
+ kind: HandoffFileKind;
+ bytes: number[];
+}> = [
+ // "%PDF-"
+ { kind: "pdf", bytes: [0x25, 0x50, 0x44, 0x46, 0x2d] },
+ // "PK\x03\x04": the local file header every OOXML/ODF container (.docx,
+ // .xlsx, .pptx, .odt, ...) starts with. All four bytes, so a text body that
+ // merely begins with "PK" is not mistaken for an archive.
+ { kind: "zip", bytes: [0x50, 0x4b, 0x03, 0x04] },
+ // OLE2/CFB header shared by legacy Office files (.doc, .xls, .ppt).
+ { kind: "ole", bytes: [0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1] },
+];
+
+const HANDOFF_SNIFF_BYTES = Math.max(
+ ...HANDOFF_FILE_SIGNATURES.map(s => s.bytes.length),
+);
+
+// Headers that already declare a ZIP archive. A ZIP signature under one of
+// these confirms a real archive rather than exposing a mislabeled Office
+// file, so it keeps the unsupported-file rejection instead of being sent to
+// the document parser.
+const ZIP_ARCHIVE_MEDIA_TYPES = new Set([
+ "application/zip",
+ "application/x-zip-compressed",
+ "application/java-archive",
+]);
+
+function matchHandoffSignature(
+ head: ArrayLike<number>,
+): HandoffFileKind | null {
+ for (const { kind, bytes } of HANDOFF_FILE_SIGNATURES) {
+ if (head.length < bytes.length) continue;
+ let matches = true;
+ for (let i = 0; i < bytes.length; i++) {
+ if (head[i] !== bytes[i]) {
+ matches = false;
+ break;
+ }
+ }
+ if (matches) return kind;
+ }
+ return null;
+}
+
+/**
+ * Identifies a handoff's file type from its leading bytes. A captured
+ * download arrives base64-encoded in `file.content` (4 characters encode 3
+ * bytes, so a short prefix covers the longest signature without decoding the
+ * file); a body fire-engine returned as text sits in `content` and is
+ * compared character by character.
+ */
+function sniffHandoffFileKind(
+ feRes: FireEngineCheckStatusSuccess | undefined,
+): HandoffFileKind | null {
+ if (!feRes) return null;
+ if (feRes.file?.content !== undefined) {
+ const chars = Math.ceil(HANDOFF_SNIFF_BYTES / 3) * 4;
+ const kind = matchHandoffSignature(
+ Buffer.from(feRes.file.content.slice(0, chars), "base64"),
+ );
+ if (kind !== null) return kind;
+ }
+ return matchHandoffSignature(
+ Array.from(feRes.content.slice(0, HANDOFF_SNIFF_BYTES), c =>
+ c.charCodeAt(0),
+ ),
+ );
+}
+
+/**
+ * Hands a sniffed file to the engine that parses it. `contentType` is the
+ * header as served (undefined without one); the document engine only uses it
+ * as an extension hint before falling back to the URL path and the
+ * converter's own detection.
+ */
+async function handoffSniffedFile(
+ kind: HandoffFileKind,
+ logger: Logger,
+ feRes: FireEngineCheckStatusSuccess | undefined,
+ contentType: string | undefined,
+ signal?: AbortSignal,
+ maxFileBytes?: number,
+): Promise<never> {
+ if (kind === "pdf") {
+ throw new AddFeatureError(
+ ["pdf"],
+ await feResToPdfPrefetch(logger, feRes, signal, maxFileBytes),
+ );
+ }
+ // OLE2 signature is shared by .doc/.xls/.ppt files
+ // Only override to application/msword if URL suggests it's a .doc file
+ const url = feRes?.url?.toLowerCase() ?? "";
+ const isDocUrl = url.endsWith(".doc") || url.includes(".doc?");
+ const effectiveContentType =
+ kind === "ole" && isDocUrl ? "application/msword" : contentType;
+ throw new AddFeatureError(
+ ["document"],
+ undefined,
+ await feResToDocumentPrefetch(logger, feRes, effectiveContentType),
+ );
+}
+
export async function specialtyScrapeCheck(
logger: Logger,
headers: Record<string, string> | undefined,
@@ -176,6 +289,17 @@ export async function specialtyScrapeCheck(
await feResToImagePrefetch(logger, feRes, headerlessImage),
);
}
+ const headerlessFile = sniffHandoffFileKind(feRes);
+ if (headerlessFile !== null) {
+ await handoffSniffedFile(
+ headerlessFile,
+ logger,
+ feRes,
+ undefined,
+ signal,
+ maxFileBytes,
+ );
+ }
logger.warn("Failed to check contentType -- was not present in headers", {
headers,
});
@@ -186,7 +310,6 @@ export async function specialtyScrapeCheck(
const isPdf =
contentType === "application/pdf" ||
contentType.startsWith("application/pdf;");
- const isOctetStream = contentType === "application/octet-stream";
// Check for document types first (before PDF to prioritize documents)
if (isDocument) {
@@ -197,40 +320,6 @@ export async function specialtyScrapeCheck(
);
}
- // Check for octet-stream with document signature
- // Modern Office files (.docx, .xlsx) are ZIP archives starting with "PK" (base64: "UEsD")
- // Legacy Office files (.doc, .xls) are OLE2/CFB files starting with D0 CF 11 E0 (base64: "0M8R4K")
- if (isOctetStream) {
- const isZipSignature =
- feRes?.file?.content?.startsWith("UEsD") ||
- feRes?.content.startsWith("PK");
- const isOleSignature =
- feRes?.file?.content?.startsWith("0M8R4K") ||
- feRes?.content.startsWith("\xD0\xCF\x11\xE0");
-
- if (isZipSignature) {
- throw new AddFeatureError(
- ["document"],
- undefined,
- await feResToDocumentPrefetch(logger, feRes, contentType),
- );
- }
- if (isOleSignature) {
- // OLE2 signature is shared by .doc/.xls/.ppt files
- // Only override to application/msword if URL suggests it's a .doc file
- const url = feRes?.url?.toLowerCase() ?? "";
- const isDocUrl = url.endsWith(".doc") || url.includes(".doc?");
- const effectiveContentType = isDocUrl
- ? "application/msword"
- : contentType;
- throw new AddFeatureError(
- ["document"],
- undefined,
- await feResToDocumentPrefetch(logger, feRes, effectiveContentType),
- );
- }
- }
-
// Check for PDF (references were already handled above the header guard).
if (isPdf) {
throw new AddFeatureError(
@@ -239,24 +328,13 @@ export async function specialtyScrapeCheck(
);
}
- // Check for octet-stream with PDF signature
- if (
- isOctetStream &&
- (feRes?.file?.content?.startsWith("JVBERi0") ||
- feRes?.content.startsWith("%PDF-"))
- ) {
- throw new AddFeatureError(
- ["pdf"],
- await feResToPdfPrefetch(logger, feRes, signal, maxFileBytes),
- );
- }
+ const mediaType = contentType.split(";")[0].trim().toLowerCase();
// Raster images are OCR'd as one-page scanned documents by the image
// engine (see engines/image). The header must claim an image or an opaque
// octet-stream AND the bytes must confirm a format FirePDF can open, so a
// mislabeled HTML error page served as image/jpeg is never sent to OCR
// and WebP/SVG/AVIF keep failing fast below.
- const mediaType = contentType.split(";")[0].trim().toLowerCase();
if (
mediaType.startsWith("image/") ||
mediaType === "application/octet-stream"
@@ -272,6 +350,27 @@ export async function specialtyScrapeCheck(
}
}
+ // The header named no parseable type, but the bytes may still: a PDF served
+ // as application/octet-stream or image/jp2, an Office file behind a generic
+ // download endpoint. The browser only hands off what it could not render,
+ // so every remaining content type is sniffed β€” after the image check above,
+ // so a real raster image still reaches OCR, and before the rejection below,
+ // so a mislabeled PDF is parsed instead of failing as an unsupported image.
+ const sniffedFile = sniffHandoffFileKind(feRes);
+ if (
+ sniffedFile !== null &&
+ !(sniffedFile === "zip" && ZIP_ARCHIVE_MEDIA_TYPES.has(mediaType))
+ ) {
+ await handoffSniffedFile(
+ sniffedFile,
+ logger,
+ feRes,
+ contentType,
+ signal,
+ maxFileBytes,
+ );
+ }
+
// Reject unsupported binary content types (unsupported images, video, audio, archives, etc.)
const unsupportedBinaryPrefixes = [
"image/",
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 2857705a9..b1e8b3138 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -24,6 +24,7 @@ import {
scrapeURLWithEngine,
shouldUseIndex,
} from "./engines";
+import { applyHandoffFeatureFlags } from "./lib/handoffFeatureFlags";
import { parseMarkdown } from "../../lib/html-to-markdown";
import { hasFormatOfType } from "../../lib/format-utils";
import {
@@ -1386,14 +1387,24 @@ export async function scrapeURL(
Array.isArray(meta.internalOptions.forceEngine))
) {
retryTracker.record("feature_toggle", error);
+ // A file handoff names the one parser that can open the file;
+ // the file flag the URL extension implied earlier gives way to it.
+ const nextFeatureFlags = applyHandoffFeatureFlags(
+ meta.featureFlags,
+ error.featureFlags,
+ );
meta.logger.debug(
"More feature flags requested by scraper: adding " +
error.featureFlags.join(", "),
- { error, existingFlags: meta.featureFlags },
- );
- meta.featureFlags = new Set(
- [...meta.featureFlags].concat(error.featureFlags),
+ {
+ error,
+ existingFlags: meta.featureFlags,
+ droppedFlags: [...meta.featureFlags].filter(
+ flag => !nextFeatureFlags.has(flag),
+ ),
+ },
);
+ meta.featureFlags = nextFeatureFlags;
if (error.pdfPrefetch) {
meta.pdfPrefetch = error.pdfPrefetch;
} else if (error.pdfPrefetch === null) {
diff --git a/apps/api/src/scraper/scrapeURL/lib/handoffFeatureFlags.test.ts b/apps/api/src/scraper/scrapeURL/lib/handoffFeatureFlags.test.ts
new file mode 100644
index 000000000..f2a0be241
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/lib/handoffFeatureFlags.test.ts
@@ -0,0 +1,46 @@
+import { describe, expect, it } from "vitest";
+import type { FeatureFlag } from "../engines";
+import { applyHandoffFeatureFlags } from "./handoffFeatureFlags";
+
+describe("applyHandoffFeatureFlags", () => {
+ it("replaces the URL-derived file flag with the parser the handoff names", () => {
+ // A .pdf URL that served a docx.
+ expect(applyHandoffFeatureFlags(new Set(["pdf"]), ["document"])).toEqual(
+ new Set(["document"]),
+ );
+ // A .docx URL that served a PDF.
+ expect(applyHandoffFeatureFlags(new Set(["document"]), ["pdf"])).toEqual(
+ new Set(["pdf"]),
+ );
+ // A .pdf URL that served a PNG.
+ expect(applyHandoffFeatureFlags(new Set(["pdf"]), ["image"])).toEqual(
+ new Set(["image"]),
+ );
+ });
+
+ it("keeps every non-file flag", () => {
+ expect(
+ applyHandoffFeatureFlags(new Set(["pdf", "location", "waitFor"]), [
+ "document",
+ ]),
+ ).toEqual(new Set(["document", "location", "waitFor"]));
+ });
+
+ it("is a plain union when the handoff names no parser", () => {
+ expect(
+ applyHandoffFeatureFlags(new Set(["pdf", "location"]), ["stealthProxy"]),
+ ).toEqual(new Set(["pdf", "location", "stealthProxy"]));
+ });
+
+ it("leaves a matching file flag in place", () => {
+ expect(
+ applyHandoffFeatureFlags(new Set(["pdf", "mobile"]), ["pdf"]),
+ ).toEqual(new Set(["pdf", "mobile"]));
+ });
+
+ it("does not mutate the request's set", () => {
+ const current = new Set<FeatureFlag>(["pdf"]);
+ applyHandoffFeatureFlags(current, ["document"]);
+ expect(current).toEqual(new Set(["pdf"]));
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/lib/handoffFeatureFlags.ts b/apps/api/src/scraper/scrapeURL/lib/handoffFeatureFlags.ts
new file mode 100644
index 000000000..ed5026058
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/lib/handoffFeatureFlags.ts
@@ -0,0 +1,44 @@
+import type { FeatureFlag } from "../engines";
+
+/**
+ * The parser flags a file handoff can name. A file is exactly one of these:
+ * the bytes (or header) that produced the handoff decide which.
+ */
+const FILE_FEATURE_FLAGS: ReadonlySet<FeatureFlag> = new Set<FeatureFlag>([
+ "pdf",
+ "document",
+ "image",
+]);
+
+/**
+ * Merges a handoff's feature flags into the request's set.
+ *
+ * A file flag the URL extension implied up front (the `pdf` of a .pdf URL
+ * that turns out to serve a docx) is dropped when the handoff names a
+ * different parser. The file flags weigh 100 each in buildFallbackList's
+ * support threshold, so keeping both would demand a parser that handles both
+ * types: with nothing else set the single-type parsers only just qualify, and
+ * any further flag (location, stealthProxy, waitFor) pushes the threshold
+ * past them, stranding a file that is already in hand.
+ */
+export function applyHandoffFeatureFlags(
+ current: ReadonlySet<FeatureFlag>,
+ added: readonly FeatureFlag[],
+): Set<FeatureFlag> {
+ const handedOff = added.filter(flag => FILE_FEATURE_FLAGS.has(flag));
+ const next = new Set<FeatureFlag>();
+ for (const flag of current) {
+ if (
+ handedOff.length > 0 &&
+ FILE_FEATURE_FLAGS.has(flag) &&
+ !handedOff.includes(flag)
+ ) {
+ continue;
+ }
+ next.add(flag);
+ }
+ for (const flag of added) {
+ next.add(flag);
+ }
+ return next;
+}
diff --git a/apps/test-site/public/mislabeled-pdf.jp2 b/apps/test-site/public/mislabeled-pdf.jp2
new file mode 100644
index 000000000..186d7ae47
--- /dev/null
+++ b/apps/test-site/public/mislabeled-pdf.jp2
@@ -0,0 +1,77 @@
+%PDF-1.3
+1 0 obj
+<< /Type /Catalog /Pages 2 0 R >>
+endobj
+2 0 obj
+<< /Type /Pages /Kids [3 0 R] /Count 1 >>
+endobj
+3 0 obj
+<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>
+endobj
+4 0 obj
+<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>
+endobj
+5 0 obj
+<< /Length 5170 >>
+stream
+BT /F1 18 Tf 72 730 Td (Firecrawl mislabeled PDF fixture) Tj ET
+BT /F1 12 Tf 72 706 Td (Served with an image content type, but the bytes are a PDF.) Tj ET
+BT /F1 9 Tf 72 680 Td (Fixture body line 01 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 667 Td (Fixture body line 02 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 654 Td (Fixture body line 03 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 641 Td (Fixture body line 04 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 628 Td (Fixture body line 05 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 615 Td (Fixture body line 06 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 602 Td (Fixture body line 07 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 589 Td (Fixture body line 08 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 576 Td (Fixture body line 09 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 563 Td (Fixture body line 10 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 550 Td (Fixture body line 11 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 537 Td (Fixture body line 12 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 524 Td (Fixture body line 13 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 511 Td (Fixture body line 14 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 498 Td (Fixture body line 15 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 485 Td (Fixture body line 16 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 472 Td (Fixture body line 17 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 459 Td (Fixture body line 18 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 446 Td (Fixture body line 19 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 433 Td (Fixture body line 20 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 420 Td (Fixture body line 21 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 407 Td (Fixture body line 22 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 394 Td (Fixture body line 23 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 381 Td (Fixture body line 24 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 368 Td (Fixture body line 25 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 355 Td (Fixture body line 26 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 342 Td (Fixture body line 27 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 329 Td (Fixture body line 28 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 316 Td (Fixture body line 29 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 303 Td (Fixture body line 30 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 290 Td (Fixture body line 31 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 277 Td (Fixture body line 32 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 264 Td (Fixture body line 33 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 251 Td (Fixture body line 34 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 238 Td (Fixture body line 35 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 225 Td (Fixture body line 36 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 212 Td (Fixture body line 37 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 199 Td (Fixture body line 38 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 186 Td (Fixture body line 39 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 173 Td (Fixture body line 40 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 160 Td (Fixture body line 41 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 147 Td (Fixture body line 42 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 134 Td (Fixture body line 43 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+BT /F1 9 Tf 72 121 Td (Fixture body line 44 of 44: the bytes are a PDF although the header says image/jp2.) Tj ET
+endstream
+endobj
+xref
+0 6
+0000000000 65535 f
+0000000009 00000 n
+0000000058 00000 n
+0000000115 00000 n
+0000000241 00000 n
+0000000311 00000 n
+trailer
+<< /Size 6 /Root 1 0 R >>
+startxref
+5533
+%%EOF
diff --git a/apps/test-site/public/tiny-image.jp2 b/apps/test-site/public/tiny-image.jp2
new file mode 100644
index 000000000..90b925bbc
Binary files /dev/null and b/apps/test-site/public/tiny-image.jp2 differ