Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): /map returning less urls with sitemap include
delong35b95906caf96
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex 6f5371ddd..0cf1f45a6 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -143,6 +143,7 @@ export class WebCrawler {limit: number,maxDepth: number,fromMap: boolean = false,+ skipRobots: boolean = false,): Promise<FilterLinksResult> {const denialReasons = new Map<string, string>();@@ -173,7 +174,7 @@ export class WebCrawler {excludes: this.excludes,includes: this.includes,allowBackwardCrawling: this.allowBackwardCrawling,- ignoreRobotsTxt: this.ignoreRobotsTxt,+ ignoreRobotsTxt: this.ignoreRobotsTxt || skipRobots,robotsTxt: this.robotsTxt,allowExternalContentLinks: this.allowExternalContentLinks,allowSubdomains: this.allowSubdomains,@@ -318,11 +319,12 @@ export class WebCrawler {}}- const isAllowed = this.ignoreRobotsTxt- ? true- : ((this.robots.isAllowed(link, "FireCrawlAgent") ||- this.robots.isAllowed(link, "FirecrawlAgent")) ??- true);+ const isAllowed =+ this.ignoreRobotsTxt || skipRobots+ ? true+ : ((this.robots.isAllowed(link, "FireCrawlAgent") ||+ this.robots.isAllowed(link, "FirecrawlAgent")) ??+ true);// Check if the link is disallowed by robots.txtif (!isAllowed) {this.logger.debug(`Link disallowed by robots.txt: ${link}`, {@@ -451,6 +453,7 @@ export class WebCrawler {leftOfLimit,this.maxCrawledDepth,fromMap,+ fromMap,);let filteredLinks = filteredLinksResult.links;leftOfLimit -= filteredLinks.length;