Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(crawler): always set expiry on sitemap links in redis
Gergő Móricz474e5a054373
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex 34d1426ad..56c46faf9 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -334,6 +334,12 @@ export class WebCrawler {count++;}+ await redisConnection.expire(+ "sitemap:" + this.jobId + ":links",+ 3600,+ "NX",+ );+return count;} catch (error) {if (error.message === "Sitemap fetch timeout") {