Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(crawler): check for more strings
Gergő Móriczc22c87ab0a9f
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex 5e91b46e7..126520e2d 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -179,7 +179,7 @@ export class WebCrawler {const isAllowed = this.ignoreRobotsTxt? true- : (this.robots.isAllowed(link, "FireCrawlAgent") ?? true);+ : ((this.robots.isAllowed(link, "FireCrawlAgent") || this.robots.isAllowed(link, "FirecrawlAgent")) ?? true);// Check if the link is disallowed by robots.txtif (!isAllowed) {this.logger.debug(`Link disallowed by robots.txt: ${link}`, {@@ -453,7 +453,7 @@ export class WebCrawler {return ignoreRobotsTxt? true: this.robots- ? (this.robots.isAllowed(url, "FireCrawlAgent") ?? true)+ ? ((this.robots.isAllowed(url, "FireCrawlAgent") || this.robots.isAllowed(url, "FirecrawlAgent")) ?? true): true;}