Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Add gzipped sitemap support (#2132)

tom3f956ef9fe54
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index e34335134..bdb44a5f7 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -14,7 +14,6 @@ import {
isUrlAllowedByRobots,
} from "../../lib/robots-txt";
import { ScrapeJobTimeoutError } from "../../lib/error";
-import { Logger } from "winston";
import { ScrapeOptions } from "../../controllers/v2/types";
import { filterLinks } from "@mendable/firecrawl-rs";
@@ -833,15 +832,17 @@ export class WebCrawler {
mock?: string,
maxAge?: number,
): Promise<number> {
- const sitemapUrl = url.endsWith(".xml")
- ? url
- : `${url}${url.endsWith("/") ? "" : "/"}sitemap.xml`;
+ const sitemapUrl =
+ url.endsWith(".xml") || url.endsWith(".xml.gz")
+ ? url
+ : `${url}${url.endsWith("/") ? "" : "/"}sitemap.xml`;
this.logger.debug("Trying to fetch sitemap links", {
method: "tryFetchSitemapLinks",
originalUrl: url,
sitemapUrl,
isXmlUrl: url.endsWith(".xml"),
+ isGzUrl: url.endsWith(".xml.gz"),
});
let sitemapCount: number = 0;
diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.ts
index 27a02a609..700b2277e 100644
--- a/apps/api/src/scraper/WebScraper/sitemap.ts
+++ b/apps/api/src/scraper/WebScraper/sitemap.ts
@@ -8,10 +8,16 @@ import { ScrapeJobTimeoutError } from "../../lib/error";
import type { ScrapeOptions } from "../../controllers/v2/types";
import { Engine } from "../scrapeURL/engines";
import { parseSitemapXml, processSitemap } from "@mendable/firecrawl-rs";
+import { gunzip } from "node:zlib";
+import { promisify } from "node:util";
+import { fetchFileToBuffer } from "../scrapeURL/engines/utils/downloadFile";
+
const useFireEngine =
process.env.FIRE_ENGINE_BETA_URL !== "" &&
process.env.FIRE_ENGINE_BETA_URL !== undefined;
+const gunzipAsync = promisify(gunzip);
+
export async function getLinksFromSitemap(
{
sitemapUrl,
@@ -46,86 +52,101 @@ export async function getLinksFromSitemap(
sitemapsHit.add(sitemapUrl);
try {
- let content: string = "";
- try {
- const shouldPrioritizeFireEngine =
- location && mode === "fire-engine" && useFireEngine;
-
- const forceEngine: Engine[] = [
- ...(maxAge > 0 ? ["index" as const] : []),
- ...(shouldPrioritizeFireEngine
- ? [
- "fire-engine;tlsclient" as const,
- "fire-engine;tlsclient;stealth" as const,
- ]
- : []),
- "fetch",
- ...(!shouldPrioritizeFireEngine &&
- mode === "fire-engine" &&
- useFireEngine
- ? [
- "fire-engine;tlsclient" as const,
- "fire-engine;tlsclient;stealth" as const,
- ]
- : []),
- ];
-
- const response = await scrapeURL(
- "sitemap;" + crawlId,
- sitemapUrl,
- scrapeOptions.parse({
- formats: ["rawHtml"],
- useMock: mock,
- maxAge,
- ...(location ? { location } : {}),
- }),
- {
- forceEngine,
- v0DisableJsDom: true,
- externalAbort: abort
- ? {
- signal: abort,
- tier: "external",
- throwable() {
- return new Error("Sitemap fetch aborted");
- },
- }
- : undefined,
- teamId: "sitemap",
- zeroDataRetention,
- },
- new CostTracking(),
- );
+ let content = "";
- if (
- response.success &&
- response.document.metadata.statusCode >= 200 &&
- response.document.metadata.statusCode < 300
- ) {
- content = response.document.rawHtml!;
- } else {
- logger.error(`Request failed for sitemap fetch`, {
- method: "getLinksFromSitemap",
- mode,
+ const isGzip = sitemapUrl.toLowerCase().endsWith(".gz");
+ if (isGzip) {
+ try {
+ const { buffer } = await fetchFileToBuffer(sitemapUrl);
+ const decompressed = await gunzipAsync(buffer);
+ content = decompressed.toString("utf-8");
+ } catch (error) {
+ logger.error("Failed to download/decompress gzip sitemap", {
sitemapUrl,
- error: response.success
- ? response.document.metadata.statusCode
- : response.error,
+ error,
});
return 0;
}
- } catch (error) {
- if (error instanceof ScrapeJobTimeoutError) {
- throw error;
- } else {
- logger.error(`Request failed for sitemap fetch`, {
- method: "getLinksFromSitemap",
- mode,
+ } else {
+ try {
+ const shouldPrioritizeFireEngine =
+ location && mode === "fire-engine" && useFireEngine;
+
+ const forceEngine: Engine[] = [
+ ...(maxAge > 0 ? ["index" as const] : []),
+ ...(shouldPrioritizeFireEngine
+ ? [
+ "fire-engine;tlsclient" as const,
+ "fire-engine;tlsclient;stealth" as const,
+ ]
+ : []),
+ "fetch",
+ ...(!shouldPrioritizeFireEngine &&
+ mode === "fire-engine" &&
+ useFireEngine
+ ? [
+ "fire-engine;tlsclient" as const,
+ "fire-engine;tlsclient;stealth" as const,
+ ]
+ : []),
+ ];
+
+ const response = await scrapeURL(
+ "sitemap;" + crawlId,
sitemapUrl,
- error,
- });
+ scrapeOptions.parse({
+ formats: ["rawHtml"],
+ useMock: mock,
+ maxAge,
+ ...(location ? { location } : {}),
+ }),
+ {
+ forceEngine,
+ v0DisableJsDom: true,
+ externalAbort: abort
+ ? {
+ signal: abort,
+ tier: "external",
+ throwable() {
+ return new Error("Sitemap fetch aborted");
+ },
+ }
+ : undefined,
+ teamId: "sitemap",
+ zeroDataRetention,
+ },
+ new CostTracking(),
+ );
- return 0;
+ if (
+ response.success &&
+ response.document.metadata.statusCode >= 200 &&
+ response.document.metadata.statusCode < 300
+ ) {
+ content = response.document.rawHtml!;
+ } else {
+ logger.error(`Request failed for sitemap fetch`, {
+ method: "getLinksFromSitemap",
+ mode,
+ sitemapUrl,
+ error: response.success
+ ? response.document.metadata.statusCode
+ : response.error,
+ });
+ return 0;
+ }
+ } catch (error) {
+ if (error instanceof ScrapeJobTimeoutError) {
+ throw error;
+ } else {
+ logger.error(`Request failed for sitemap fetch`, {
+ method: "getLinksFromSitemap",
+ mode,
+ sitemapUrl,
+ error,
+ });
+ return 0;
+ }
}
}
@@ -184,7 +205,8 @@ export async function getLinksFromSitemap(
url =>
url.loc &&
url.loc.length > 0 &&
- url.loc[0].trim().toLowerCase().endsWith(".xml"),
+ (url.loc[0].trim().toLowerCase().endsWith(".xml") ||
+ url.loc[0].trim().toLowerCase().endsWith(".xml.gz")),
)
.map(url => url.loc[0].trim());
@@ -217,6 +239,7 @@ export async function getLinksFromSitemap(
url.loc &&
url.loc.length > 0 &&
!url.loc[0].trim().toLowerCase().endsWith(".xml") &&
+ !url.loc[0].trim().toLowerCase().endsWith(".xml.gz") &&
!WebCrawler.prototype.isFile(url.loc[0].trim()),
)
.map(url => url.loc[0].trim());
@@ -246,7 +269,7 @@ export async function getLinksFromSitemap(
),
);
const results = await Promise.all(sitemapPromises);
- count += results.reduce((a, x) => a + x);
+ count += results.reduce((a, x) => a + x, 0);
} else if (instruction.action === "process") {
count += instruction.urls.length;
const h = urlsHandler(instruction.urls);