Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): cap sitemap downloads and gunzip output (#5492)
Abimael Martellec011992d1ce
diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.tsindex 2ad3bcc5f..198ba4cdc 100644--- a/apps/api/src/scraper/WebScraper/sitemap.ts+++ b/apps/api/src/scraper/WebScraper/sitemap.ts@@ -14,13 +14,13 @@ import {processSitemap,SitemapProcessingResult,} from "@mendable/firecrawl-rs";-import { gunzip } from "node:zlib";-import { promisify } from "node:util";import { fetchFileToBuffer } from "../scrapeURL/engines/utils/downloadFile";+import {+ gunzipBounded,+ MAX_GUNZIPPED_FILE_SIZE,+} from "../scrapeURL/engines/utils/gunzip";import { useIndex } from "../../services";-const gunzipAsync = promisify(gunzip);-export async function getLinksFromSitemap({sitemapUrl,@@ -62,11 +62,13 @@ export async function getLinksFromSitemap(const isGzip = sitemapUrl.toLowerCase().endsWith(".gz");if (isGzip) {try {- const { buffer } = await fetchFileToBuffer(sitemapUrl, false, {- headers,- signal: abort,- });- const decompressed = await gunzipAsync(buffer);+ const { buffer } = await fetchFileToBuffer(+ sitemapUrl,+ false,+ { headers, signal: abort },+ MAX_GUNZIPPED_FILE_SIZE,+ );+ const decompressed = await gunzipBounded(buffer);content = decompressed.toString("utf-8");} catch (error) {logger.error("Failed to download/decompress gzip sitemap", {diff --git a/apps/api/src/scraper/crawler/sitemap.ts b/apps/api/src/scraper/crawler/sitemap.tsindex 970b79c2d..f96fac663 100644--- a/apps/api/src/scraper/crawler/sitemap.ts+++ b/apps/api/src/scraper/crawler/sitemap.ts@@ -10,8 +10,10 @@ import {SitemapProcessingResult,} from "@mendable/firecrawl-rs";import { fetchFileToBuffer } from "../scrapeURL/engines/utils/downloadFile";-import { gunzip } from "node:zlib";-import { promisify } from "node:util";+import {+ gunzipBounded,+ MAX_GUNZIPPED_FILE_SIZE,+} from "../scrapeURL/engines/utils/gunzip";import { SitemapError } from "../../lib/error";import { useIndex } from "../../services";@@ -30,13 +32,26 @@ type SitemapData = {sitemaps: URL[];};-const gunzipAsync = promisify(gunzip);-async function _getSitemapXMLGZ(options: SitemapScrapeOptions,): Promise<string> {- const { buffer } = await fetchFileToBuffer(options.url);- const decompressed = await gunzipAsync(buffer);+ const { buffer } = await fetchFileToBuffer(+ options.url,+ false,+ undefined,+ MAX_GUNZIPPED_FILE_SIZE,+ );+ let decompressed: Buffer;+ try {+ decompressed = await gunzipBounded(buffer);+ } catch (error) {+ throw new SitemapError(+ error instanceof RangeError+ ? "The gzipped sitemap inflates past the 50 MB limit."+ : "The gzipped sitemap could not be decompressed.",+ error,+ );+ }return decompressed.toString("utf-8");}diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex 1bd425f05..87086e10d 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -35,6 +35,7 @@ import {largePdfLimitBytes,} from "../pdf/fire-pdf/by-reference";import { PDF_DOWNLOAD_MAX_FILE_SIZE } from "../pdf/types";+import { MAX_GUNZIPPED_FILE_SIZE } from "../utils/gunzip";import { config } from "../../../../config";/** The handoff additionally requires the inbound allowlist bucket to be@@ -65,8 +66,6 @@ import { scheduleABComparison } from "../../../../services/ab-test-comparison";/** Default wait (ms) before running the branding script when user did not set waitFor. Lets the page settle so DOM/images are ready and reduces JS errors. */const BRANDING_DEFAULT_WAIT_MS = 2000;-const MAX_GUNZIPPED_FILE_SIZE = 50 * 1024 * 1024; // 50MB-// This function does not take `Meta` on purpose. It may not access any// meta values to construct the request -- that must be done by the// `scrapeURLWithFireEngine*` functions.diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/gunzip.test.ts b/apps/api/src/scraper/scrapeURL/engines/utils/gunzip.test.tsnew file mode 100644index 000000000..304504b96--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/utils/gunzip.test.ts@@ -0,0 +1,45 @@+import { gzipSync } from "node:zlib";+import { gunzipBounded, MAX_GUNZIPPED_FILE_SIZE } from "./gunzip";++const CAP = 64 * 1024;++describe("gunzipBounded", () => {+ it("inflates output under the cap", async () => {+ const original = Buffer.from("<urlset>".repeat(100));+ const inflated = await gunzipBounded(gzipSync(original), CAP);+ expect(inflated.equals(original)).toBe(true);+ });++ it("inflates output exactly at the cap", async () => {+ const original = Buffer.alloc(CAP, 1);+ const inflated = await gunzipBounded(gzipSync(original), CAP);+ expect(inflated.equals(original)).toBe(true);+ });++ it("rejects output one byte over the cap", async () => {+ const bomb = gzipSync(Buffer.alloc(CAP + 1));+ await expect(gunzipBounded(bomb, CAP)).rejects.toThrow(RangeError);+ });++ it("rejects output over an explicit cap", async () => {+ const bomb = gzipSync(Buffer.alloc(1024 * 1024));+ await expect(gunzipBounded(bomb, CAP)).rejects.toThrow(RangeError);+ });++ describe("ceiling", () => {+ // A few kilobytes of gzip that inflate just past the ceiling.+ const bomb = gzipSync(Buffer.alloc(MAX_GUNZIPPED_FILE_SIZE + 1));++ it("rejects output over the default cap", async () => {+ expect(bomb.length).toBeLessThan(1024 * 1024);+ await expect(gunzipBounded(bomb)).rejects.toThrow(RangeError);+ });++ it.each([MAX_GUNZIPPED_FILE_SIZE * 2, Infinity, NaN])(+ "never raises the cap past the ceiling (%s)",+ async cap => {+ await expect(gunzipBounded(bomb, cap)).rejects.toThrow(RangeError);+ },+ );+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/gunzip.ts b/apps/api/src/scraper/scrapeURL/engines/utils/gunzip.tsnew file mode 100644index 000000000..5b07e3a2b--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/utils/gunzip.ts@@ -0,0 +1,23 @@+import { gunzip } from "node:zlib";+import { promisify } from "node:util";++// Upper bound on inflated output. A few kilobytes of gzip can inflate to+// gigabytes, so every gunzip of caller-controlled bytes must carry a cap.+export const MAX_GUNZIPPED_FILE_SIZE = 50 * 1024 * 1024; // 50MB++const gunzipAsync = promisify(gunzip);++/**+ * Inflates `buffer`, rejecting with a RangeError (ERR_BUFFER_TOO_LARGE) once+ * the output would exceed `maxOutputLength`.+ */+export async function gunzipBounded(+ buffer: Buffer,+ maxOutputLength: number = MAX_GUNZIPPED_FILE_SIZE,+): Promise<Buffer> {+ // Callers may tighten the cap but never raise it past the ceiling.+ const bounded = Number.isFinite(maxOutputLength)+ ? Math.min(maxOutputLength, MAX_GUNZIPPED_FILE_SIZE)+ : MAX_GUNZIPPED_FILE_SIZE;+ return gunzipAsync(buffer, { maxOutputLength: bounded });+}