Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): honor robotsUserAgent in native link filter (#3405)
firecrawl-spring[bot]9ae36e527707
diff --git a/apps/api/native/src/crawler.rs b/apps/api/native/src/crawler.rsindex d301481ad..3dbb1b2b4 100644--- a/apps/api/native/src/crawler.rs+++ b/apps/api/native/src/crawler.rs@@ -33,6 +33,7 @@ pub struct FilterLinksCall {pub allow_backward_crawling: bool,pub ignore_robots_txt: bool,pub robots_txt: String,+ pub robots_user_agent: Option<String>,pub allow_external_content_links: bool,pub allow_subdomains: bool,}@@ -53,6 +54,7 @@ pub struct FilterUrlCall {pub excludes: Vec<String>,pub ignore_robots_txt: bool,pub robots_txt: String,+ pub robots_user_agent: Option<String>,pub allow_external_content_links: bool,pub allow_subdomains: bool,}@@ -227,6 +229,22 @@ fn is_external_main_page(url_str: &str) -> bool {}}+fn build_robot(+ ignore_robots_txt: bool,+ robots_txt: &str,+ robots_user_agent: Option<&str>,+) -> Option<Robot> {+ if ignore_robots_txt || robots_txt.is_empty() {+ return None;+ }+ if let Some(ua) = robots_user_agent {+ return Robot::new(ua, robots_txt.as_bytes()).ok();+ }+ Robot::new("FireCrawlAgent", robots_txt.as_bytes())+ .ok()+ .or_else(|| Robot::new("FirecrawlAgent", robots_txt.as_bytes()).ok())+}+fn _filter_links(data: FilterLinksCall) -> std::result::Result<FilterLinksResult, String> {let limit = data.limit.map_or(usize::MAX, |x| x.max(0) as usize);if limit == 0 {@@ -252,13 +270,11 @@ fn _filter_links(data: FilterLinksCall) -> std::result::Result<FilterLinksResult.filter_map(|i| Regex::new(i).ok()).collect();- let robot = if !data.ignore_robots_txt && !data.robots_txt.is_empty() {- Robot::new("FireCrawlAgent", data.robots_txt.as_bytes())- .ok()- .or_else(|| Robot::new("FirecrawlAgent", data.robots_txt.as_bytes()).ok())- } else {- None- };+ let robot = build_robot(+ data.ignore_robots_txt,+ &data.robots_txt,+ data.robots_user_agent.as_deref(),+ );let mut result_links = Vec::new();let mut denial_reasons = HashMap::new();@@ -458,13 +474,11 @@ fn _filter_url(data: FilterUrlCall) -> std::result::Result<FilterUrlResult, Stri.filter_map(|e| Regex::new(e).ok()).collect();- let robot = if !data.ignore_robots_txt && !data.robots_txt.is_empty() {- Robot::new("FireCrawlAgent", data.robots_txt.as_bytes())- .ok()- .or_else(|| Robot::new("FirecrawlAgent", data.robots_txt.as_bytes()).ok())- } else {- None- };+ let robot = build_robot(+ data.ignore_robots_txt,+ &data.robots_txt,+ data.robots_user_agent.as_deref(),+ );if is_internal_link(&url, &base_url) {// INTERNAL LINKS@@ -895,6 +909,7 @@ mod tests {allow_backward_crawling: true,allow_external_content_links: false,allow_subdomains: false,+ robots_user_agent: None,};let result = _filter_links(data).unwrap();@@ -929,6 +944,7 @@ mod tests {allow_backward_crawling: true,allow_external_content_links: false,allow_subdomains: false,+ robots_user_agent: None,};let result = _filter_links(data);@@ -958,6 +974,7 @@ mod tests {allow_backward_crawling: true,allow_external_content_links: false,allow_subdomains: false,+ robots_user_agent: None,};let result = _filter_links(data);@@ -985,6 +1002,7 @@ mod tests {allow_backward_crawling: true,allow_external_content_links: false,allow_subdomains: false,+ robots_user_agent: None,};let result = _filter_links(data);@@ -1015,6 +1033,7 @@ mod tests {allow_backward_crawling: true,allow_external_content_links: false,allow_subdomains: true,+ robots_user_agent: None,};let result = _filter_links(data).unwrap();@@ -1042,6 +1061,44 @@ mod tests {);}+ #[test]+ fn test_filter_links_honors_custom_robots_user_agent() {+ // robots.txt allows the default FireCrawlAgent but blocks CustomBot. Without+ // a custom user-agent the link is allowed; with `robots_user_agent` wired+ // through it must be filtered.+ let robots_txt = "User-agent: *\nAllow: /\n\nUser-agent: CustomBot\nDisallow: /";++ let base_call = |ua: Option<String>| FilterLinksCall {+ links: vec!["https://example.com/page".to_string()],+ limit: Some(10),+ includes: vec![],+ excludes: vec![],+ ignore_robots_txt: false,+ robots_txt: robots_txt.to_string(),+ max_depth: 10,+ base_url: "https://example.com".to_string(),+ initial_url: "https://example.com".to_string(),+ regex_on_full_url: false,+ allow_backward_crawling: true,+ allow_external_content_links: false,+ allow_subdomains: false,+ robots_user_agent: ua,+ };++ let default_result = _filter_links(base_call(None)).unwrap();+ assert_eq!(default_result.links, vec!["https://example.com/page"]);++ let custom_result = _filter_links(base_call(Some("CustomBot".to_string()))).unwrap();+ assert!(custom_result.links.is_empty());+ assert_eq!(+ custom_result+ .denial_reasons+ .get("https://example.com/page")+ .unwrap(),+ "ROBOTS_TXT"+ );+ }+#[test]fn test_is_file() {assert!(is_file("test.png"));diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex 0b2b49061..28105c72e 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -195,6 +195,7 @@ export class WebCrawler {allowBackwardCrawling: this.allowBackwardCrawling,ignoreRobotsTxt: this.ignoreRobotsTxt || skipRobots,robotsTxt: this.robotsTxt,+ robotsUserAgent: this.robotsUserAgent,allowExternalContentLinks: this.allowExternalContentLinks,allowSubdomains: this.allowSubdomains,});@@ -667,6 +668,7 @@ export class WebCrawler {excludes: this.excludes,ignoreRobotsTxt: this.ignoreRobotsTxt,robotsTxt: this.robotsTxt,+ robotsUserAgent: this.robotsUserAgent,allowExternalContentLinks: this.allowExternalContentLinks,allowSubdomains: this.allowSubdomains,});