Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat: transparent gzip decompression (#3898)

tomsideguide80da82307c36
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
index c967e4014..2e513f769 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -26,6 +26,7 @@ import {
ProxySelectionError,
} from "../../error";
import * as Sentry from "@sentry/node";
+import { gunzipSync } from "node:zlib";
import { specialtyScrapeCheck } from "../utils/specialtyHandler";
import { fireEngineDelete } from "./delete";
import { MockState } from "../../lib/mock";
@@ -43,6 +44,8 @@ import { createHash } from "node:crypto";
/** Default wait (ms) before running the branding script when user did not set waitFor. Lets the page settle so DOM/images are ready and reduces JS errors. */
const BRANDING_DEFAULT_WAIT_MS = 2000;
+const MAX_GUNZIPPED_FILE_SIZE = 50 * 1024 * 1024; // 50MB
+
// This function does not take `Meta` on purpose. It may not access any
// meta values to construct the request -- that must be done by the
// `scrapeURLWithFireEngine*` functions.
@@ -212,7 +215,21 @@ async function performFireEngineScrape<
if (status.file) {
const content = status.file.content;
delete status.file;
- status.content = Buffer.from(content, "base64").toString("utf8"); // TODO: handle other encodings via Content-Type tag
+ let buffer = Buffer.from(content, "base64");
+ // transparently decompress gzipped content
+ if (buffer.length >= 2 && buffer[0] === 0x1f && buffer[1] === 0x8b) {
+ try {
+ buffer = gunzipSync(buffer, {
+ maxOutputLength: MAX_GUNZIPPED_FILE_SIZE,
+ });
+ } catch (error) {
+ if (error instanceof RangeError) {
+ throw new UnsupportedFileError("File exceeds size limit");
+ }
+ throw new UnsupportedFileError("Failed to decompress gzip content");
+ }
+ }
+ status.content = buffer.toString("utf8"); // TODO: handle other encodings via Content-Type tag
}
fireEngineDelete(
diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
index 3e31e0a72..464218f12 100644
--- a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
@@ -163,7 +163,6 @@ export async function specialtyScrapeCheck(
"audio/",
"application/zip",
"application/x-tar",
- "application/gzip",
"application/x-rar",
"application/x-7z",
"application/wasm",