Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(crawler): check for more strings

Gergő Móriczc22c87ab0a9f
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 5e91b46e7..126520e2d 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -179,7 +179,7 @@ export class WebCrawler {
const isAllowed = this.ignoreRobotsTxt
? true
- : (this.robots.isAllowed(link, "FireCrawlAgent") ?? true);
+ : ((this.robots.isAllowed(link, "FireCrawlAgent") || this.robots.isAllowed(link, "FirecrawlAgent")) ?? true);
// Check if the link is disallowed by robots.txt
if (!isAllowed) {
this.logger.debug(`Link disallowed by robots.txt: ${link}`, {
@@ -453,7 +453,7 @@ export class WebCrawler {
return ignoreRobotsTxt
? true
: this.robots
- ? (this.robots.isAllowed(url, "FireCrawlAgent") ?? true)
+ ? ((this.robots.isAllowed(url, "FireCrawlAgent") || this.robots.isAllowed(url, "FirecrawlAgent")) ?? true)
: true;
}