Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Reapply "fix(v1/crawl): always use sitemap"
Nicolas445fc432e9e4
diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex e0883fa8e..e49cfe90d 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -85,9 +85,7 @@ export async function crawlController(await saveCrawl(id, sc);- const sitemap = sc.crawlerOptions.ignoreSitemap- ? null- : await crawler.tryGetSitemap();+ const sitemap = await crawler.tryGetSitemap();if (sitemap !== null && sitemap.length > 0) {let jobPriority = 20;