Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(scrape-worker): cap sitemaps per crawl at 20 to avoid queue flooding
Gergő Móricz322996347bf7
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 02efe206c..1ddcf1ce1 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -713,6 +713,16 @@ async function addKickoffSitemapJob(sc: StoredCrawl,logger: Logger,) {+ // TEMP: max 20 sitemaps per crawl+ if (+ (await redisEvictConnection.scard(+ "crawl:" + sourceJob.data.crawl_id + ":sitemaps",+ )) >= 20+ ) {+ logger.debug("Sitemap limit reached, skipping...", { sitemap: sitemapUrl });+ return;+ }+const sitemapLocked =(await redisEvictConnection.sadd("crawl:" + sourceJob.data.crawl_id + ":sitemaps",