Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(scrape-worker): cap sitemaps per crawl at 20 to avoid queue flooding

Gergő Móricz322996347bf7
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index 02efe206c..1ddcf1ce1 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -713,6 +713,16 @@ async function addKickoffSitemapJob(
sc: StoredCrawl,
logger: Logger,
) {
+ // TEMP: max 20 sitemaps per crawl
+ if (
+ (await redisEvictConnection.scard(
+ "crawl:" + sourceJob.data.crawl_id + ":sitemaps",
+ )) >= 20
+ ) {
+ logger.debug("Sitemap limit reached, skipping...", { sitemap: sitemapUrl });
+ return;
+ }
+
const sitemapLocked =
(await redisEvictConnection.sadd(
"crawl:" + sourceJob.data.crawl_id + ":sitemaps",