Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): parse PDFs under 4 KiB with pdf-parse (#4507)

Abimael Martell90312e40788e
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/pdfParse.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/pdfParse.test.ts
new file mode 100644
index 000000000..ac0cd99c6
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/pdfParse.test.ts
@@ -0,0 +1,152 @@
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { scrapePDFWithParsePDF } from "../pdfParse";
+
+/**
+ * Minimal valid PDF: one Helvetica text line per entry of each page's `lines`.
+ * ASCII only, so xref offsets can be computed from string lengths. No fixtures
+ * checked into the repo.
+ */
+function makePdf(pages: string[][]): Buffer {
+ const kids = pages.map((_, i) => `${4 + i * 2} 0 R`).join(" ");
+ const objects: [number, string][] = [
+ [1, "<< /Type /Catalog /Pages 2 0 R >>"],
+ [2, `<< /Type /Pages /Kids [${kids}] /Count ${pages.length} >>`],
+ [3, "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>"],
+ ];
+ pages.forEach((lines, i) => {
+ const pageId = 4 + i * 2;
+ const contentId = pageId + 1;
+ const ops = lines
+ .map((line, j) => `${j === 0 ? "" : "0 -14 Td "}(${line}) Tj`)
+ .join(" ");
+ const content = `BT /F1 12 Tf 72 720 Td ${ops} ET`;
+ objects.push([
+ pageId,
+ `<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >> /Contents ${contentId} 0 R >>`,
+ ]);
+ objects.push([
+ contentId,
+ `<< /Length ${content.length} >>\nstream\n${content}\nendstream`,
+ ]);
+ });
+
+ let out = "%PDF-1.4\n";
+ const offsets: number[] = [];
+ for (const [id, body] of objects) {
+ offsets[id] = out.length;
+ out += `${id} 0 obj\n${body}\nendobj\n`;
+ }
+ const xrefPos = out.length;
+ out += `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n`;
+ for (let id = 1; id <= objects.length; id++) {
+ out += `${offsets[id]}`.padStart(10, "0") + " 00000 n \n";
+ }
+ out += `trailer\n<< /Size ${objects.length + 1} /Root 1 0 R >>\nstartxref\n${xrefPos}\n%%EOF\n`;
+ // Buffer.alloc never uses Node's shared Buffer pool, so the only pooled copy
+ // of these bytes can be the one pdf.js makes (see skewBufferPool).
+ const pdf = Buffer.alloc(out.length);
+ pdf.write(out, "latin1");
+ return pdf;
+}
+
+/**
+ * pdf-parse's bundled pdf.js clones a Buffer input with `new Buffer(...)`,
+ * which Node serves from its shared 8 KiB pool for anything under 4 KiB, and
+ * then reads that slice's ArrayBuffer at absolute offsets. The parse only
+ * succeeds by accident when the clone lands at the very start of a fresh pool,
+ * so make sure the next pooled allocation of `size` bytes cannot.
+ */
+function skewBufferPool(size: number) {
+ const probe = Buffer.allocUnsafe(8);
+ const remaining = Buffer.poolSize - (probe.byteOffset + 8);
+ if (remaining < size + 1024) {
+ // Too little room left: roll over to a fresh pool now, leaving its first
+ // bytes occupied so the clone starts at a non-zero offset.
+ Buffer.allocUnsafe(remaining + 1);
+ }
+}
+
+function makeMeta() {
+ const logger = {
+ debug: vi.fn(),
+ info: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ };
+ const meta = {
+ id: "pdf-parse-test",
+ url: "https://example.com/file.pdf",
+ logger,
+ } as any;
+ return { meta, logger };
+}
+
+describe("scrapePDFWithParsePDF", () => {
+ let dir: string;
+
+ beforeAll(() => {
+ dir = mkdtempSync(path.join(tmpdir(), "pdf-parse-test-"));
+ });
+
+ afterAll(() => {
+ rmSync(dir, { recursive: true, force: true });
+ });
+
+ // Regression: every PDF under ~4 KiB used to fail with "bad XRef entry".
+ it("extracts text from a PDF smaller than 4 KiB", async () => {
+ const pdf = makePdf([["Hello from a tiny PDF"]]);
+ expect(pdf.length).toBeLessThan(4096);
+ const filePath = path.join(dir, "small.pdf");
+ writeFileSync(filePath, pdf);
+
+ const { meta, logger } = makeMeta();
+ skewBufferPool(pdf.length);
+ const result = await scrapePDFWithParsePDF(meta, filePath);
+
+ expect(result.markdown).toContain("Hello from a tiny PDF");
+ expect(result.html).toContain("Hello from a tiny PDF");
+ expect(logger.info).toHaveBeenCalledWith(
+ "pdfParse succeeded",
+ expect.objectContaining({ numPages: 1 }),
+ );
+ });
+
+ it("extracts every page of a PDF larger than 4 KiB", async () => {
+ const pages = Array.from({ length: 5 }, (_, p) =>
+ Array.from(
+ { length: 10 },
+ (_, l) =>
+ `The quick brown fox jumps over the lazy dog on page ${p + 1}, line ${l + 1}.`,
+ ),
+ );
+ const pdf = makePdf(pages);
+ expect(pdf.length).toBeGreaterThan(4096);
+ const filePath = path.join(dir, "large.pdf");
+ writeFileSync(filePath, pdf);
+
+ const { meta, logger } = makeMeta();
+ const result = await scrapePDFWithParsePDF(meta, filePath);
+
+ for (let p = 1; p <= pages.length; p++) {
+ expect(result.markdown).toContain(`on page ${p}, line 10.`);
+ }
+ expect(logger.info).toHaveBeenCalledWith(
+ "pdfParse succeeded",
+ expect.objectContaining({ numPages: pages.length }),
+ );
+ });
+
+ it("rethrows and logs when the file is not a PDF", async () => {
+ const filePath = path.join(dir, "not-a-pdf.pdf");
+ writeFileSync(filePath, "<html><body>Not Found</body></html>");
+
+ const { meta, logger } = makeMeta();
+ await expect(scrapePDFWithParsePDF(meta, filePath)).rejects.toThrow();
+ expect(logger.error).toHaveBeenCalledWith(
+ "pdfParse failed",
+ expect.objectContaining({ error: expect.anything() }),
+ );
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/pdfParse.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/pdfParse.ts
index 62fc1b211..e07324735 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/pdfParse.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/pdfParse.ts
@@ -12,7 +12,19 @@ export async function scrapePDFWithParsePDF(
try {
const startedAt = Date.now();
- const result = await PdfParse(await readFile(tempFilePath));
+ const file = await readFile(tempFilePath);
+ // pdf-parse's bundled pdf.js (1.10) clones its input with
+ // `new value.constructor(value)`. For a Buffer that is `new Buffer(...)`,
+ // which Node serves from its shared 8 KiB pool for anything under 4 KiB,
+ // so the clone is a slice with a non-zero byteOffset — and pdf.js then
+ // reads `bytes.buffer` at absolute offsets (Stream.makeSubStream). Every
+ // PDF under ~4 KiB therefore failed with "bad XRef entry" unless the slice
+ // happened to land at the start of a fresh pool. A Uint8Array view over
+ // the same memory (no copy) turns that clone into `new Uint8Array(...)`,
+ // which owns its ArrayBuffer. (@types/pdf-parse declares Buffer, but
+ // pdf.js accepts any Uint8Array.)
+ const bytes = new Uint8Array(file.buffer, file.byteOffset, file.byteLength);
+ const result = await PdfParse(bytes as Buffer);
const durationMs = Date.now() - startedAt;
const escaped = escapeHtml(result.text);