Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): cap sitemap downloads and gunzip output (#5492)

Abimael Martellec011992d1ce
diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.ts
index 2ad3bcc5f..198ba4cdc 100644
--- a/apps/api/src/scraper/WebScraper/sitemap.ts
+++ b/apps/api/src/scraper/WebScraper/sitemap.ts
@@ -14,13 +14,13 @@ import {
processSitemap,
SitemapProcessingResult,
} from "@mendable/firecrawl-rs";
-import { gunzip } from "node:zlib";
-import { promisify } from "node:util";
import { fetchFileToBuffer } from "../scrapeURL/engines/utils/downloadFile";
+import {
+ gunzipBounded,
+ MAX_GUNZIPPED_FILE_SIZE,
+} from "../scrapeURL/engines/utils/gunzip";
import { useIndex } from "../../services";
-const gunzipAsync = promisify(gunzip);
-
export async function getLinksFromSitemap(
{
sitemapUrl,
@@ -62,11 +62,13 @@ export async function getLinksFromSitemap(
const isGzip = sitemapUrl.toLowerCase().endsWith(".gz");
if (isGzip) {
try {
- const { buffer } = await fetchFileToBuffer(sitemapUrl, false, {
- headers,
- signal: abort,
- });
- const decompressed = await gunzipAsync(buffer);
+ const { buffer } = await fetchFileToBuffer(
+ sitemapUrl,
+ false,
+ { headers, signal: abort },
+ MAX_GUNZIPPED_FILE_SIZE,
+ );
+ const decompressed = await gunzipBounded(buffer);
content = decompressed.toString("utf-8");
} catch (error) {
logger.error("Failed to download/decompress gzip sitemap", {
diff --git a/apps/api/src/scraper/crawler/sitemap.ts b/apps/api/src/scraper/crawler/sitemap.ts
index 970b79c2d..f96fac663 100644
--- a/apps/api/src/scraper/crawler/sitemap.ts
+++ b/apps/api/src/scraper/crawler/sitemap.ts
@@ -10,8 +10,10 @@ import {
SitemapProcessingResult,
} from "@mendable/firecrawl-rs";
import { fetchFileToBuffer } from "../scrapeURL/engines/utils/downloadFile";
-import { gunzip } from "node:zlib";
-import { promisify } from "node:util";
+import {
+ gunzipBounded,
+ MAX_GUNZIPPED_FILE_SIZE,
+} from "../scrapeURL/engines/utils/gunzip";
import { SitemapError } from "../../lib/error";
import { useIndex } from "../../services";
@@ -30,13 +32,26 @@ type SitemapData = {
sitemaps: URL[];
};
-const gunzipAsync = promisify(gunzip);
-
async function _getSitemapXMLGZ(
options: SitemapScrapeOptions,
): Promise<string> {
- const { buffer } = await fetchFileToBuffer(options.url);
- const decompressed = await gunzipAsync(buffer);
+ const { buffer } = await fetchFileToBuffer(
+ options.url,
+ false,
+ undefined,
+ MAX_GUNZIPPED_FILE_SIZE,
+ );
+ let decompressed: Buffer;
+ try {
+ decompressed = await gunzipBounded(buffer);
+ } catch (error) {
+ throw new SitemapError(
+ error instanceof RangeError
+ ? "The gzipped sitemap inflates past the 50 MB limit."
+ : "The gzipped sitemap could not be decompressed.",
+ error,
+ );
+ }
return decompressed.toString("utf-8");
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
index 1bd425f05..87086e10d 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -35,6 +35,7 @@ import {
largePdfLimitBytes,
} from "../pdf/fire-pdf/by-reference";
import { PDF_DOWNLOAD_MAX_FILE_SIZE } from "../pdf/types";
+import { MAX_GUNZIPPED_FILE_SIZE } from "../utils/gunzip";
import { config } from "../../../../config";
/** The handoff additionally requires the inbound allowlist bucket to be
@@ -65,8 +66,6 @@ import { scheduleABComparison } from "../../../../services/ab-test-comparison";
/** Default wait (ms) before running the branding script when user did not set waitFor. Lets the page settle so DOM/images are ready and reduces JS errors. */
const BRANDING_DEFAULT_WAIT_MS = 2000;
-const MAX_GUNZIPPED_FILE_SIZE = 50 * 1024 * 1024; // 50MB
-
// This function does not take `Meta` on purpose. It may not access any
// meta values to construct the request -- that must be done by the
// `scrapeURLWithFireEngine*` functions.
diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/gunzip.test.ts b/apps/api/src/scraper/scrapeURL/engines/utils/gunzip.test.ts
new file mode 100644
index 000000000..304504b96
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/utils/gunzip.test.ts
@@ -0,0 +1,45 @@
+import { gzipSync } from "node:zlib";
+import { gunzipBounded, MAX_GUNZIPPED_FILE_SIZE } from "./gunzip";
+
+const CAP = 64 * 1024;
+
+describe("gunzipBounded", () => {
+ it("inflates output under the cap", async () => {
+ const original = Buffer.from("<urlset>".repeat(100));
+ const inflated = await gunzipBounded(gzipSync(original), CAP);
+ expect(inflated.equals(original)).toBe(true);
+ });
+
+ it("inflates output exactly at the cap", async () => {
+ const original = Buffer.alloc(CAP, 1);
+ const inflated = await gunzipBounded(gzipSync(original), CAP);
+ expect(inflated.equals(original)).toBe(true);
+ });
+
+ it("rejects output one byte over the cap", async () => {
+ const bomb = gzipSync(Buffer.alloc(CAP + 1));
+ await expect(gunzipBounded(bomb, CAP)).rejects.toThrow(RangeError);
+ });
+
+ it("rejects output over an explicit cap", async () => {
+ const bomb = gzipSync(Buffer.alloc(1024 * 1024));
+ await expect(gunzipBounded(bomb, CAP)).rejects.toThrow(RangeError);
+ });
+
+ describe("ceiling", () => {
+ // A few kilobytes of gzip that inflate just past the ceiling.
+ const bomb = gzipSync(Buffer.alloc(MAX_GUNZIPPED_FILE_SIZE + 1));
+
+ it("rejects output over the default cap", async () => {
+ expect(bomb.length).toBeLessThan(1024 * 1024);
+ await expect(gunzipBounded(bomb)).rejects.toThrow(RangeError);
+ });
+
+ it.each([MAX_GUNZIPPED_FILE_SIZE * 2, Infinity, NaN])(
+ "never raises the cap past the ceiling (%s)",
+ async cap => {
+ await expect(gunzipBounded(bomb, cap)).rejects.toThrow(RangeError);
+ },
+ );
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/gunzip.ts b/apps/api/src/scraper/scrapeURL/engines/utils/gunzip.ts
new file mode 100644
index 000000000..5b07e3a2b
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/utils/gunzip.ts
@@ -0,0 +1,23 @@
+import { gunzip } from "node:zlib";
+import { promisify } from "node:util";
+
+// Upper bound on inflated output. A few kilobytes of gzip can inflate to
+// gigabytes, so every gunzip of caller-controlled bytes must carry a cap.
+export const MAX_GUNZIPPED_FILE_SIZE = 50 * 1024 * 1024; // 50MB
+
+const gunzipAsync = promisify(gunzip);
+
+/**
+ * Inflates `buffer`, rejecting with a RangeError (ERR_BUFFER_TOO_LARGE) once
+ * the output would exceed `maxOutputLength`.
+ */
+export async function gunzipBounded(
+ buffer: Buffer,
+ maxOutputLength: number = MAX_GUNZIPPED_FILE_SIZE,
+): Promise<Buffer> {
+ // Callers may tighten the cap but never raise it past the ceiling.
+ const bounded = Number.isFinite(maxOutputLength)
+ ? Math.min(maxOutputLength, MAX_GUNZIPPED_FILE_SIZE)
+ : MAX_GUNZIPPED_FILE_SIZE;
+ return gunzipAsync(buffer, { maxOutputLength: bounded });
+}