Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api/crawler/sitemap): bump sitemap limit (#2079)
Gergő Móricz68d3b844152f
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex 8bf82e8fe..272238f43 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -14,6 +14,8 @@ import { filterLinks } from "../../lib/crawler";import { fetchRobotsTxt, createRobotsChecker, isUrlAllowedByRobots } from "../../lib/robots-txt";import { ScrapeJobTimeoutError } from "../../lib/error";+export const SITEMAP_LIMIT = 100;+export interface FilterResult {allowed: boolean;url?: string;@@ -864,7 +866,7 @@ export class WebCrawler {}}- if (this.sitemapsHit.size >= 20) {+ if (this.sitemapsHit.size >= SITEMAP_LIMIT) {this.logger.warn("Sitemap limit hit!", { crawlId: this.jobId, url: this.baseUrl });}diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.tsindex bb89024cb..7efcd86a9 100644--- a/apps/api/src/scraper/WebScraper/sitemap.ts+++ b/apps/api/src/scraper/WebScraper/sitemap.ts@@ -1,5 +1,5 @@import { parseStringPromise } from "xml2js";-import { WebCrawler } from "./crawler";+import { WebCrawler, SITEMAP_LIMIT } from "./crawler";import { scrapeURL } from "../scrapeURL";import { scrapeOptions } from "../../controllers/v2/types";import type { Logger } from "winston";@@ -9,6 +9,8 @@ import { ScrapeJobTimeoutError } from "../../lib/error";const useFireEngine =process.env.FIRE_ENGINE_BETA_URL !== "" &&process.env.FIRE_ENGINE_BETA_URL !== undefined;++export async function getLinksFromSitemap({sitemapUrl,@@ -29,7 +31,7 @@ export async function getLinksFromSitemap(abort?: AbortSignal,mock?: string,): Promise<number> {- if (sitemapsHit.size >= 20) {+ if (sitemapsHit.size >= SITEMAP_LIMIT) {return 0;}