Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api/crawler/sitemap): bump sitemap limit (#2079)

Gergő Móricz68d3b844152f
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 8bf82e8fe..272238f43 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -14,6 +14,8 @@ import { filterLinks } from "../../lib/crawler";
import { fetchRobotsTxt, createRobotsChecker, isUrlAllowedByRobots } from "../../lib/robots-txt";
import { ScrapeJobTimeoutError } from "../../lib/error";
+export const SITEMAP_LIMIT = 100;
+
export interface FilterResult {
allowed: boolean;
url?: string;
@@ -864,7 +866,7 @@ export class WebCrawler {
}
}
- if (this.sitemapsHit.size >= 20) {
+ if (this.sitemapsHit.size >= SITEMAP_LIMIT) {
this.logger.warn("Sitemap limit hit!", { crawlId: this.jobId, url: this.baseUrl });
}
diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.ts
index bb89024cb..7efcd86a9 100644
--- a/apps/api/src/scraper/WebScraper/sitemap.ts
+++ b/apps/api/src/scraper/WebScraper/sitemap.ts
@@ -1,5 +1,5 @@
import { parseStringPromise } from "xml2js";
-import { WebCrawler } from "./crawler";
+import { WebCrawler, SITEMAP_LIMIT } from "./crawler";
import { scrapeURL } from "../scrapeURL";
import { scrapeOptions } from "../../controllers/v2/types";
import type { Logger } from "winston";
@@ -9,6 +9,8 @@ import { ScrapeJobTimeoutError } from "../../lib/error";
const useFireEngine =
process.env.FIRE_ENGINE_BETA_URL !== "" &&
process.env.FIRE_ENGINE_BETA_URL !== undefined;
+
+
export async function getLinksFromSitemap(
{
sitemapUrl,
@@ -29,7 +31,7 @@ export async function getLinksFromSitemap(
abort?: AbortSignal,
mock?: string,
): Promise<number> {
- if (sitemapsHit.size >= 20) {
+ if (sitemapsHit.size >= SITEMAP_LIMIT) {
return 0;
}