Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Update sitemap.ts
Nicolasac5e1fc194bd
diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.tsindex d93403c88..756cd7651 100644--- a/apps/api/src/scraper/WebScraper/sitemap.ts+++ b/apps/api/src/scraper/WebScraper/sitemap.ts@@ -23,7 +23,7 @@ export async function getLinksFromSitemap(const response = await axios.get(sitemapUrl, { timeout: axiosTimeout });content = response.data;} else if (mode === 'fire-engine') {- const response = await scrapWithFireEngine({ url: sitemapUrl, fireEngineOptions: { engine:"chrome-cdp" } });+ const response = await scrapWithFireEngine({ url: sitemapUrl, fireEngineOptions: { engine:"playwright" } });content = response.html;}} catch (error) {