Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(pdf): catch marked.parse stack overflow in PDF pipeline (#3352)

Abimael Martell3d55317ff851
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/markdownToHtml.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/markdownToHtml.test.ts
new file mode 100644
index 000000000..b5f460ace
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/markdownToHtml.test.ts
@@ -0,0 +1,67 @@
+import { safeMarkdownToHtml } from "../markdownToHtml";
+
+const noopLogger = {
+ warn: jest.fn(),
+} as any;
+
+describe("safeMarkdownToHtml", () => {
+ afterEach(() => {
+ jest.restoreAllMocks();
+ noopLogger.warn.mockClear();
+ });
+
+ it("converts simple markdown to HTML", async () => {
+ const html = await safeMarkdownToHtml("# Hello", noopLogger, "test-1");
+ expect(html).toContain("<h1>");
+ expect(html).toContain("Hello");
+ });
+
+ it("does not throw on pathologically deep or large markdown", async () => {
+ // 50,000 nested blockquotes
+ const deep = "> ".repeat(50_000) + "content";
+ const htmlDeep = await safeMarkdownToHtml(deep, noopLogger, "test-deep");
+ expect(typeof htmlDeep).toBe("string");
+ expect(htmlDeep.length).toBeGreaterThan(0);
+
+ // ~200KB markdown table
+ const row = "| " + "cell | ".repeat(10) + "\n";
+ const header = row + "| " + "--- | ".repeat(10) + "\n";
+ const table = header + row.repeat(5_000);
+ expect(table.length).toBeGreaterThan(200_000);
+
+ const htmlTable = await safeMarkdownToHtml(table, noopLogger, "test-table");
+ expect(typeof htmlTable).toBe("string");
+ expect(htmlTable.length).toBeGreaterThan(0);
+ });
+
+ it("falls back to escaped <pre> and logs a warning when marked.parse throws", async () => {
+ // monkey-patch required since marked exports are non-configurable (can't use jest.spyOn)
+ const markedModule = require("marked");
+ const originalParse = markedModule.parse;
+ markedModule.parse = () => {
+ throw new RangeError("Maximum call stack size exceeded");
+ };
+
+ try {
+ const input = '<script>alert("xss")</script> & "quotes" \'apos\'';
+ const html = await safeMarkdownToHtml(input, noopLogger, "test-escape");
+
+ expect(html.startsWith("<pre>")).toBe(true);
+ expect(html).toContain("&lt;script&gt;");
+ expect(html).toContain("&amp;");
+ expect(html).toContain("&quot;quotes&quot;");
+ expect(html).toContain("&#39;apos&#39;");
+ expect(html).not.toContain("<script>");
+
+ expect(noopLogger.warn).toHaveBeenCalledWith(
+ expect.stringContaining("marked.parse failed"),
+ expect.objectContaining({
+ scrapeId: "test-escape",
+ markdownLength: expect.any(Number),
+ }),
+ );
+ } finally {
+ markedModule.parse = originalParse;
+ }
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
index 4f72a9d80..70ea3d60e 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
@@ -2,8 +2,8 @@ import { Meta } from "../..";
import { config } from "../../../../config";
import { robustFetch } from "../../lib/fetch";
import { z } from "zod";
-import * as marked from "marked";
import type { PDFProcessorResult } from "./types";
+import { safeMarkdownToHtml } from "./markdownToHtml";
import {
getPdfResultFromCache,
savePdfResultToCache,
@@ -78,7 +78,7 @@ export async function scrapePDFWithFirePDF(
const processorResult = {
markdown: resp.markdown,
- html: await marked.parse(resp.markdown, { async: true }),
+ html: await safeMarkdownToHtml(resp.markdown, logger, meta.id),
};
if (!maxPages && !meta.internalOptions.zeroDataRetention) {
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index 679e24389..8640924d3 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -1,8 +1,8 @@
import { Meta } from "../..";
import { config } from "../../../../config";
import { EngineScrapeResult } from "..";
-import * as marked from "marked";
import { downloadFile, fetchFileToBuffer } from "../utils/downloadFile";
+import { safeMarkdownToHtml } from "./markdownToHtml";
import {
PDFAntibotError,
PDFInsufficientTimeError,
@@ -300,7 +300,11 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
}
if (eligible && pdfResult.markdown) {
- const html = await marked.parse(pdfResult.markdown, { async: true });
+ const html = await safeMarkdownToHtml(
+ pdfResult.markdown,
+ logger,
+ meta.id,
+ );
result = { markdown: pdfResult.markdown, html };
}
} catch (error) {
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/markdownToHtml.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/markdownToHtml.ts
new file mode 100644
index 000000000..20214f676
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/markdownToHtml.ts
@@ -0,0 +1,24 @@
+import * as marked from "marked";
+import type { Logger } from "winston";
+
+export async function safeMarkdownToHtml(
+ markdown: string,
+ logger: Logger,
+ scrapeId: string,
+): Promise<string> {
+ try {
+ return await marked.parse(markdown, { async: true });
+ } catch (e) {
+ logger.warn("marked.parse failed, falling back to <pre> wrapper", {
+ error: e,
+ scrapeId,
+ markdownLength: markdown.length,
+ });
+ return `<pre>${markdown
+ .replace(/&/g, "&amp;")
+ .replace(/</g, "&lt;")
+ .replace(/>/g, "&gt;")
+ .replace(/"/g, "&quot;")
+ .replace(/'/g, "&#39;")}</pre>`;
+ }
+}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts
index cd2d889ef..554a5460c 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts
@@ -1,7 +1,7 @@
import { Meta } from "../..";
import { config } from "../../../../config";
-import * as marked from "marked";
import { robustFetch } from "../../lib/fetch";
+import { safeMarkdownToHtml } from "./markdownToHtml";
import { z } from "zod";
import path from "node:path";
import {
@@ -188,7 +188,7 @@ export async function scrapePDFWithRunPodMU(
const processorResult = {
markdown: result.markdown,
- html: await marked.parse(result.markdown, { async: true }),
+ html: await safeMarkdownToHtml(result.markdown, meta.logger, meta.id),
};
if (!meta.internalOptions.zeroDataRetention) {