Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): /map returning less urls with sitemap include

delong35b95906caf96
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 6f5371ddd..0cf1f45a6 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -143,6 +143,7 @@ export class WebCrawler {
limit: number,
maxDepth: number,
fromMap: boolean = false,
+ skipRobots: boolean = false,
): Promise<FilterLinksResult> {
const denialReasons = new Map<string, string>();
@@ -173,7 +174,7 @@ export class WebCrawler {
excludes: this.excludes,
includes: this.includes,
allowBackwardCrawling: this.allowBackwardCrawling,
- ignoreRobotsTxt: this.ignoreRobotsTxt,
+ ignoreRobotsTxt: this.ignoreRobotsTxt || skipRobots,
robotsTxt: this.robotsTxt,
allowExternalContentLinks: this.allowExternalContentLinks,
allowSubdomains: this.allowSubdomains,
@@ -318,11 +319,12 @@ export class WebCrawler {
}
}
- const isAllowed = this.ignoreRobotsTxt
- ? true
- : ((this.robots.isAllowed(link, "FireCrawlAgent") ||
- this.robots.isAllowed(link, "FirecrawlAgent")) ??
- true);
+ const isAllowed =
+ this.ignoreRobotsTxt || skipRobots
+ ? true
+ : ((this.robots.isAllowed(link, "FireCrawlAgent") ||
+ this.robots.isAllowed(link, "FirecrawlAgent")) ??
+ true);
// Check if the link is disallowed by robots.txt
if (!isAllowed) {
this.logger.debug(`Link disallowed by robots.txt: ${link}`, {
@@ -451,6 +453,7 @@ export class WebCrawler {
leftOfLimit,
this.maxCrawledDepth,
fromMap,
+ fromMap,
);
let filteredLinks = filteredLinksResult.links;
leftOfLimit -= filteredLinks.length;