Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: transparent gzip decompression (#3898)
tomsideguide80da82307c36
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex c967e4014..2e513f769 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -26,6 +26,7 @@ import {ProxySelectionError,} from "../../error";import * as Sentry from "@sentry/node";+import { gunzipSync } from "node:zlib";import { specialtyScrapeCheck } from "../utils/specialtyHandler";import { fireEngineDelete } from "./delete";import { MockState } from "../../lib/mock";@@ -43,6 +44,8 @@ import { createHash } from "node:crypto";/** Default wait (ms) before running the branding script when user did not set waitFor. Lets the page settle so DOM/images are ready and reduces JS errors. */const BRANDING_DEFAULT_WAIT_MS = 2000;+const MAX_GUNZIPPED_FILE_SIZE = 50 * 1024 * 1024; // 50MB+// This function does not take `Meta` on purpose. It may not access any// meta values to construct the request -- that must be done by the// `scrapeURLWithFireEngine*` functions.@@ -212,7 +215,21 @@ async function performFireEngineScrape<if (status.file) {const content = status.file.content;delete status.file;- status.content = Buffer.from(content, "base64").toString("utf8"); // TODO: handle other encodings via Content-Type tag+ let buffer = Buffer.from(content, "base64");+ // transparently decompress gzipped content+ if (buffer.length >= 2 && buffer[0] === 0x1f && buffer[1] === 0x8b) {+ try {+ buffer = gunzipSync(buffer, {+ maxOutputLength: MAX_GUNZIPPED_FILE_SIZE,+ });+ } catch (error) {+ if (error instanceof RangeError) {+ throw new UnsupportedFileError("File exceeds size limit");+ }+ throw new UnsupportedFileError("Failed to decompress gzip content");+ }+ }+ status.content = buffer.toString("utf8"); // TODO: handle other encodings via Content-Type tag}fireEngineDelete(diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.tsindex 3e31e0a72..464218f12 100644--- a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts+++ b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts@@ -163,7 +163,6 @@ export async function specialtyScrapeCheck("audio/","application/zip","application/x-tar",- "application/gzip","application/x-rar","application/x-7z","application/wasm",