Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): sitemap max age for recursive sitemaps
delong34831da0b695f
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex da4f6b1ff..acd5d4aee 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -17,6 +17,7 @@ import { ScrapeOptions } from "../../controllers/v2/types";import { filterLinks, filterUrl } from "@mendable/firecrawl-rs";export const SITEMAP_LIMIT = 100;+const SITEMAP_MAX_AGE = 48 * 60 * 60 * 1000;interface FilterResult {allowed: boolean;@@ -494,7 +495,6 @@ export class WebCrawler {);});- const maxAge = 48 * 60 * 60 * 1000;try {const robotsSitemaps = this.robots.getSitemaps();this.logger.debug("Attempting to fetch sitemap links", {@@ -512,10 +512,16 @@ export class WebCrawler {_urlsHandler,abort,mock,- maxAge,+ SITEMAP_MAX_AGE,),...robotsSitemaps.map(x =>- this.tryFetchSitemapLinks(x, _urlsHandler, abort, mock, maxAge),+ this.tryFetchSitemapLinks(+ x,+ _urlsHandler,+ abort,+ mock,+ SITEMAP_MAX_AGE,+ ),),]).then(results => results.reduce((a, x) => a + x, 0)),timeoutPromise,@@ -714,6 +720,7 @@ export class WebCrawler {method: "tryFetchSitemapLinks",originalUrl: url,sitemapUrl,+ maxAge,isXmlUrl: url.endsWith(".xml"),isGzUrl: url.endsWith(".xml.gz"),});diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.tsindex 700b2277e..348358121 100644--- a/apps/api/src/scraper/WebScraper/sitemap.ts+++ b/apps/api/src/scraper/WebScraper/sitemap.ts@@ -7,10 +7,16 @@ import { CostTracking } from "../../lib/cost-tracking";import { ScrapeJobTimeoutError } from "../../lib/error";import type { ScrapeOptions } from "../../controllers/v2/types";import { Engine } from "../scrapeURL/engines";-import { parseSitemapXml, processSitemap } from "@mendable/firecrawl-rs";+import {+ ParsedSitemap,+ parseSitemapXml,+ processSitemap,+ SitemapProcessingResult,+} from "@mendable/firecrawl-rs";import { gunzip } from "node:zlib";import { promisify } from "node:util";import { fetchFileToBuffer } from "../scrapeURL/engines/utils/downloadFile";+import { useIndex } from "../../services";const useFireEngine =process.env.FIRE_ENGINE_BETA_URL !== "" &&@@ -73,7 +79,7 @@ export async function getLinksFromSitemap(location && mode === "fire-engine" && useFireEngine;const forceEngine: Engine[] = [- ...(maxAge > 0 ? ["index" as const] : []),+ ...(maxAge > 0 && useIndex ? ["index" as const] : []),...(shouldPrioritizeFireEngine? ["fire-engine;tlsclient" as const,@@ -150,7 +156,7 @@ export async function getLinksFromSitemap(}}- let instructions;+ let instructions: SitemapProcessingResult;try {instructions = await processSitemap(content);} catch (error) {@@ -163,7 +169,7 @@ export async function getLinksFromSitemap(},);- let parsed;+ let parsed: ParsedSitemap;try {parsed = await parseSitemapXml(content);} catch (parseError) {@@ -181,14 +187,21 @@ export async function getLinksFromSitemap(const root = parsed.urlset || parsed.sitemapindex;let count = 0;- if (root && root.sitemap) {+ if (root && "sitemap" in root && root.sitemap) {const sitemapUrls = root.sitemap.filter(sitemap => sitemap.loc && sitemap.loc.length > 0).map(sitemap => sitemap.loc[0].trim());const sitemapPromises: Promise<number>[] = sitemapUrls.map(sitemapUrl =>getLinksFromSitemap(- { sitemapUrl, urlsHandler, mode, zeroDataRetention, location },+ {+ sitemapUrl,+ urlsHandler,+ mode,+ zeroDataRetention,+ location,+ maxAge,+ },logger,crawlId,sitemapsHit,@@ -199,7 +212,7 @@ export async function getLinksFromSitemap(const results = await Promise.all(sitemapPromises);count = results.reduce((a, x) => a + x);- } else if (root && root.url) {+ } else if (root && "url" in root && root.url) {const xmlSitemaps: string[] = root.url.filter(url =>@@ -219,6 +232,7 @@ export async function getLinksFromSitemap(mode,zeroDataRetention,location,+ maxAge,},logger,crawlId,@@ -260,7 +274,14 @@ export async function getLinksFromSitemap(const sitemapPromises: Promise<number>[] = instruction.urls.map(sitemapUrl =>getLinksFromSitemap(- { sitemapUrl, urlsHandler, mode, zeroDataRetention, location },+ {+ sitemapUrl,+ urlsHandler,+ mode,+ zeroDataRetention,+ location,+ maxAge,+ },logger,crawlId,sitemapsHit,@@ -268,6 +289,7 @@ export async function getLinksFromSitemap(mock,),);+const results = await Promise.all(sitemapPromises);count += results.reduce((a, x) => a + x, 0);} else if (instruction.action === "process") {diff --git a/apps/api/src/scraper/crawler/sitemap.ts b/apps/api/src/scraper/crawler/sitemap.tsindex 9a458bc48..c24bdc4ba 100644--- a/apps/api/src/scraper/crawler/sitemap.ts+++ b/apps/api/src/scraper/crawler/sitemap.ts@@ -4,11 +4,15 @@ import { logger as _logger } from "../../lib/logger";import { Engine } from "../scrapeURL/engines";import { scrapeURL } from "../scrapeURL";import { CostTracking } from "../../lib/cost-tracking";-import { processSitemap } from "@mendable/firecrawl-rs";+import {+ processSitemap,+ SitemapProcessingResult,+} from "@mendable/firecrawl-rs";import { fetchFileToBuffer } from "../scrapeURL/engines/utils/downloadFile";import { gunzip } from "node:zlib";import { promisify } from "node:util";import { SitemapError } from "../../lib/error";+import { useIndex } from "../../services";const useFireEngine =process.env.FIRE_ENGINE_BETA_URL !== "" &&@@ -48,7 +52,7 @@ async function getSitemapXML(options: SitemapScrapeOptions): Promise<string> {options.location && options.location.country !== "us-generic";const forceEngine: Engine[] = [- ...(options.maxAge > 0 ? ["index" as const] : []),+ ...(options.maxAge > 0 && useIndex ? ["index" as const] : []),...(isLocationSpecified && useFireEngine? ["fire-engine;tlsclient" as const,@@ -120,7 +124,7 @@ export async function scrapeSitemap(logger.info("Processing sitemap");- let instructions;+ let instructions: SitemapProcessingResult;try {instructions = await processSitemap(xml);} catch (error) {