Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(pdf): catch marked.parse stack overflow in PDF pipeline (#3352)
Abimael Martell3d55317ff851
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/markdownToHtml.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/markdownToHtml.test.tsnew file mode 100644index 000000000..b5f460ace--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/markdownToHtml.test.ts@@ -0,0 +1,67 @@+import { safeMarkdownToHtml } from "../markdownToHtml";++const noopLogger = {+ warn: jest.fn(),+} as any;++describe("safeMarkdownToHtml", () => {+ afterEach(() => {+ jest.restoreAllMocks();+ noopLogger.warn.mockClear();+ });++ it("converts simple markdown to HTML", async () => {+ const html = await safeMarkdownToHtml("# Hello", noopLogger, "test-1");+ expect(html).toContain("<h1>");+ expect(html).toContain("Hello");+ });++ it("does not throw on pathologically deep or large markdown", async () => {+ // 50,000 nested blockquotes+ const deep = "> ".repeat(50_000) + "content";+ const htmlDeep = await safeMarkdownToHtml(deep, noopLogger, "test-deep");+ expect(typeof htmlDeep).toBe("string");+ expect(htmlDeep.length).toBeGreaterThan(0);++ // ~200KB markdown table+ const row = "| " + "cell | ".repeat(10) + "\n";+ const header = row + "| " + "--- | ".repeat(10) + "\n";+ const table = header + row.repeat(5_000);+ expect(table.length).toBeGreaterThan(200_000);++ const htmlTable = await safeMarkdownToHtml(table, noopLogger, "test-table");+ expect(typeof htmlTable).toBe("string");+ expect(htmlTable.length).toBeGreaterThan(0);+ });++ it("falls back to escaped <pre> and logs a warning when marked.parse throws", async () => {+ // monkey-patch required since marked exports are non-configurable (can't use jest.spyOn)+ const markedModule = require("marked");+ const originalParse = markedModule.parse;+ markedModule.parse = () => {+ throw new RangeError("Maximum call stack size exceeded");+ };++ try {+ const input = '<script>alert("xss")</script> & "quotes" \'apos\'';+ const html = await safeMarkdownToHtml(input, noopLogger, "test-escape");++ expect(html.startsWith("<pre>")).toBe(true);+ expect(html).toContain("<script>");+ expect(html).toContain("&");+ expect(html).toContain(""quotes"");+ expect(html).toContain("'apos'");+ expect(html).not.toContain("<script>");++ expect(noopLogger.warn).toHaveBeenCalledWith(+ expect.stringContaining("marked.parse failed"),+ expect.objectContaining({+ scrapeId: "test-escape",+ markdownLength: expect.any(Number),+ }),+ );+ } finally {+ markedModule.parse = originalParse;+ }+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.tsindex 4f72a9d80..70ea3d60e 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts@@ -2,8 +2,8 @@ import { Meta } from "../..";import { config } from "../../../../config";import { robustFetch } from "../../lib/fetch";import { z } from "zod";-import * as marked from "marked";import type { PDFProcessorResult } from "./types";+import { safeMarkdownToHtml } from "./markdownToHtml";import {getPdfResultFromCache,savePdfResultToCache,@@ -78,7 +78,7 @@ export async function scrapePDFWithFirePDF(const processorResult = {markdown: resp.markdown,- html: await marked.parse(resp.markdown, { async: true }),+ html: await safeMarkdownToHtml(resp.markdown, logger, meta.id),};if (!maxPages && !meta.internalOptions.zeroDataRetention) {diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 679e24389..8640924d3 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -1,8 +1,8 @@import { Meta } from "../..";import { config } from "../../../../config";import { EngineScrapeResult } from "..";-import * as marked from "marked";import { downloadFile, fetchFileToBuffer } from "../utils/downloadFile";+import { safeMarkdownToHtml } from "./markdownToHtml";import {PDFAntibotError,PDFInsufficientTimeError,@@ -300,7 +300,11 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {}if (eligible && pdfResult.markdown) {- const html = await marked.parse(pdfResult.markdown, { async: true });+ const html = await safeMarkdownToHtml(+ pdfResult.markdown,+ logger,+ meta.id,+ );result = { markdown: pdfResult.markdown, html };}} catch (error) {diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/markdownToHtml.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/markdownToHtml.tsnew file mode 100644index 000000000..20214f676--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/markdownToHtml.ts@@ -0,0 +1,24 @@+import * as marked from "marked";+import type { Logger } from "winston";++export async function safeMarkdownToHtml(+ markdown: string,+ logger: Logger,+ scrapeId: string,+): Promise<string> {+ try {+ return await marked.parse(markdown, { async: true });+ } catch (e) {+ logger.warn("marked.parse failed, falling back to <pre> wrapper", {+ error: e,+ scrapeId,+ markdownLength: markdown.length,+ });+ return `<pre>${markdown+ .replace(/&/g, "&")+ .replace(/</g, "<")+ .replace(/>/g, ">")+ .replace(/"/g, """)+ .replace(/'/g, "'")}</pre>`;+ }+}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.tsindex cd2d889ef..554a5460c 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/runpodMU.ts@@ -1,7 +1,7 @@import { Meta } from "../..";import { config } from "../../../../config";-import * as marked from "marked";import { robustFetch } from "../../lib/fetch";+import { safeMarkdownToHtml } from "./markdownToHtml";import { z } from "zod";import path from "node:path";import {@@ -188,7 +188,7 @@ export async function scrapePDFWithRunPodMU(const processorResult = {markdown: result.markdown,- html: await marked.parse(result.markdown, { async: true }),+ html: await safeMarkdownToHtml(result.markdown, meta.logger, meta.id),};if (!meta.internalOptions.zeroDataRetention) {