Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(pdf) Check for %PDF magic bytes on Pdf Engine (#2915)
Abimael Martellc0d0e5020e4a
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/isPdfBuffer.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/isPdfBuffer.test.tsnew file mode 100644index 000000000..18b328329--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/isPdfBuffer.test.ts@@ -0,0 +1,86 @@+import { isPdfBuffer } from "../pdfUtils";++describe("isPdfBuffer", () => {+ it("detects a standard PDF-1.4 header", () => {+ const buf = Buffer.from("%PDF-1.4 rest of file...");+ expect(isPdfBuffer(buf)).toBe(true);+ });++ it("detects a PDF-2.0 header", () => {+ const buf = Buffer.from("%PDF-2.0\n");+ expect(isPdfBuffer(buf)).toBe(true);+ });++ it("detects %PDF with a UTF-8 BOM prefix", () => {+ const bom = Buffer.from([0xef, 0xbb, 0xbf]);+ const pdf = Buffer.from("%PDF-1.7");+ expect(isPdfBuffer(Buffer.concat([bom, pdf]))).toBe(true);+ });++ it("detects %PDF preceded by whitespace/newlines", () => {+ const buf = Buffer.from(" \n\r\n%PDF-1.5 data...");+ expect(isPdfBuffer(buf)).toBe(true);+ });++ it("detects %PDF preceded by junk bytes within 1KB", () => {+ const junk = Buffer.alloc(500, 0x00);+ const pdf = Buffer.from("%PDF-1.4 rest...");+ expect(isPdfBuffer(Buffer.concat([junk, pdf]))).toBe(true);+ });++ it("rejects %PDF that appears after the 1KB window", () => {+ const padding = Buffer.alloc(1024, 0x20); // 1024 spaces+ const pdf = Buffer.from("%PDF-1.4");+ expect(isPdfBuffer(Buffer.concat([padding, pdf]))).toBe(false);+ });++ it("rejects HTML content", () => {+ const buf = Buffer.from(+ "<!DOCTYPE html><html><body>Not a PDF</body></html>",+ );+ expect(isPdfBuffer(buf)).toBe(false);+ });++ it("rejects plain text", () => {+ const buf = Buffer.from("Hello, this is just a text file.");+ expect(isPdfBuffer(buf)).toBe(false);+ });++ it("rejects an empty buffer", () => {+ expect(isPdfBuffer(Buffer.alloc(0))).toBe(false);+ });++ it("rejects a buffer shorter than the magic bytes", () => {+ const buf = Buffer.from("%PD");+ expect(isPdfBuffer(buf)).toBe(false);+ });++ it("rejects a buffer containing only part of the magic", () => {+ const buf = Buffer.from("%PD\x00\x00\x00");+ expect(isPdfBuffer(buf)).toBe(false);+ });++ it("rejects a 404 HTML error page", () => {+ const buf = Buffer.from(+ "<html><head><title>404 Not Found</title></head><body>Not Found</body></html>",+ );+ expect(isPdfBuffer(buf)).toBe(false);+ });++ it("rejects JSON response", () => {+ const buf = Buffer.from('{"error": "not found"}');+ expect(isPdfBuffer(buf)).toBe(false);+ });++ it("handles exactly 4-byte %PDF buffer", () => {+ const buf = Buffer.from("%PDF");+ expect(isPdfBuffer(buf)).toBe(true);+ });++ it("detects %PDF at the boundary of the 1KB window", () => {+ // Place %PDF so it ends exactly at byte 1024+ const padding = Buffer.alloc(1020, 0x41); // 1020 'A's+ const pdf = Buffer.from("%PDF");+ expect(isPdfBuffer(Buffer.concat([padding, pdf]))).toBe(true);+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 74d20f672..450d0837f 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -11,7 +11,7 @@ import {RemoveFeatureError,EngineUnsuccessfulError,} from "../../error";-import { readFile, unlink } from "node:fs/promises";+import { open, readFile, unlink } from "node:fs/promises";import type { Response } from "undici";import { AbortManagerThrownError } from "../../lib/abortManager";import {@@ -26,6 +26,7 @@ import type { PDFProcessorResult } from "./types";import { scrapePDFWithRunPodMU } from "./runpodMU";import { scrapePDFWithParsePDF } from "./pdfParse";import { captureExceptionWithZdrCheck } from "../../../../services/sentry";+import { isPdfBuffer, PDF_SNIFF_WINDOW } from "./pdfUtils";/** Check if the PDF is eligible for Rust extraction, returning a rejection reason or null. */function getIneligibleReason(@@ -68,9 +69,8 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {},);- const ct = file.response.headers.get("Content-Type");- if (ct && !ct.includes("application/pdf")) {- // if downloaded file wasn't a PDF+ if (!isPdfBuffer(file.buffer)) {+ // downloaded content isn't a valid PDFif (meta.pdfPrefetch === undefined) {// for non-PDF URLs, this is expected, not anti-botif (!meta.featureFlags.has("pdf")) {@@ -110,22 +110,30 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {);try {- if ((response as any).headers) {- // if downloadFile was used- const r: Response = response as any;- const ct = r.headers.get("Content-Type");- if (ct && !ct.includes("application/pdf")) {- // if downloaded file wasn't a PDF- if (meta.pdfPrefetch === undefined) {- // for non-PDF URLs, this is expected, not anti-bot- if (!meta.featureFlags.has("pdf")) {- throw new EngineUnsuccessfulError("pdf");- } else {- throw new PDFAntibotError();- }+ // Validate the downloaded file is actually a PDF by checking magic bytes+ const header = Buffer.alloc(PDF_SNIFF_WINDOW);+ const fh = await open(tempFilePath, "r");+ let headerBytesRead: number;+ try {+ ({ bytesRead: headerBytesRead } = await fh.read(+ header,+ 0,+ PDF_SNIFF_WINDOW,+ 0,+ ));+ } finally {+ await fh.close();+ }++ if (!isPdfBuffer(header.subarray(0, headerBytesRead))) {+ if (meta.pdfPrefetch === undefined) {+ if (!meta.featureFlags.has("pdf")) {+ throw new EngineUnsuccessfulError("pdf");} else {- throw new PDFPrefetchFailed();+ throw new PDFAntibotError();}+ } else {+ throw new PDFPrefetchFailed();}}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/pdfUtils.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/pdfUtils.tsnew file mode 100644index 000000000..653de1ac0--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/pdfUtils.ts@@ -0,0 +1,9 @@+export const PDF_SNIFF_WINDOW = 1024;++const PDF_MAGIC = Buffer.from("%PDF");++/** Check if a buffer contains the %PDF magic bytes within the first 1KB. */+export function isPdfBuffer(buf: Buffer): boolean {+ const window = buf.subarray(0, Math.min(buf.length, PDF_SNIFF_WINDOW));+ return window.includes(PDF_MAGIC);+}