Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(crawler): replace robotstxt library with texting_robots for ENG-3016 (#1895)
devin-ai-integration[bot]e7e2c02d5bf9
diff --git a/apps/api/sharedLibs/crawler/Cargo.lock b/apps/api/sharedLibs/crawler/Cargo.lockindex 3412e19ce..8a753c488 100644--- a/apps/api/sharedLibs/crawler/Cargo.lock+++ b/apps/api/sharedLibs/crawler/Cargo.lock@@ -11,16 +11,33 @@ dependencies = ["memchr",]+[[package]]+name = "anyhow"+version = "1.0.98"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487"++[[package]]+name = "bstr"+version = "0.2.17"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223"+dependencies = [+ "lazy_static",+ "memchr",+ "regex-automata 0.1.10",+]+[[package]]name = "crawler"version = "0.1.0"dependencies = ["libc","regex",- "robotstxt","roxmltree","serde","serde_json",+ "texting_robots","url",]@@ -157,6 +174,12 @@ version = "1.0.15"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"+[[package]]+name = "lazy_static"+version = "1.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"+[[package]]name = "libc"version = "0.2.174"@@ -175,6 +198,22 @@ version = "2.7.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0"+[[package]]+name = "minimal-lexical"+version = "0.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"++[[package]]+name = "nom"+version = "7.1.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"+dependencies = [+ "memchr",+ "minimal-lexical",+]+[[package]]name = "percent-encoding"version = "2.3.1"@@ -216,10 +255,16 @@ checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"dependencies = ["aho-corasick","memchr",- "regex-automata",+ "regex-automata 0.4.9","regex-syntax",]+[[package]]+name = "regex-automata"+version = "0.1.10"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"+[[package]]name = "regex-automata"version = "0.4.9"@@ -237,12 +282,6 @@ version = "0.8.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"-[[package]]-name = "robotstxt"-version = "0.3.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "bbc52377db80e3fec3a2c748ca603b8b6cacdd34ff89ff4b742a635361d4b4a7"-[[package]]name = "roxmltree"version = "0.20.0"@@ -321,6 +360,42 @@ dependencies = ["syn",]+[[package]]+name = "texting_robots"+version = "0.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5b82a718a28dda2e67ad6e0464597b58eae39e2e4d0451e03d1028d71e81bb4a"+dependencies = [+ "anyhow",+ "bstr",+ "lazy_static",+ "nom",+ "percent-encoding",+ "regex",+ "thiserror",+ "url",+]++[[package]]+name = "thiserror"+version = "1.0.69"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"+dependencies = [+ "thiserror-impl",+]++[[package]]+name = "thiserror-impl"+version = "1.0.69"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]+[[package]]name = "tinystr"version = "0.8.1"diff --git a/apps/api/sharedLibs/crawler/Cargo.toml b/apps/api/sharedLibs/crawler/Cargo.tomlindex 342789129..1e43b72e9 100644--- a/apps/api/sharedLibs/crawler/Cargo.toml+++ b/apps/api/sharedLibs/crawler/Cargo.toml@@ -9,7 +9,7 @@ serde = { version = "1.0", features = ["derive"] }serde_json = "1.0"url = "2.5.4"regex = "1.11.1"-robotstxt = "0.3.0"+texting_robots = "0.2.2"roxmltree = "0.20"[lib]diff --git a/apps/api/sharedLibs/crawler/src/lib.rs b/apps/api/sharedLibs/crawler/src/lib.rsindex aec7935a7..0ee510f98 100644--- a/apps/api/sharedLibs/crawler/src/lib.rs+++ b/apps/api/sharedLibs/crawler/src/lib.rs@@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize};use std::ffi::{CStr, CString};use url::Url;use regex::Regex;-use robotstxt::DefaultMatcher;+use texting_robots::Robot;#[derive(Deserialize)]struct FilterLinksCall {@@ -165,15 +165,27 @@ fn _filter_links(data: FilterLinksCall) -> Result<FilterLinksResult, Box<dyn std}if !data.ignore_robots_txt {- let mut matcher = DefaultMatcher::default();- let allowed = matcher.allowed_by_robots(&data.robots_txt, vec![- "FireCrawlAgent",- "FirecrawlAgent",- ], url.as_str());-- if !allowed {- denial_reasons.insert(link.clone(), "ROBOTS_TXT".to_string());- return false;+ match Robot::new("FireCrawlAgent", data.robots_txt.as_bytes()) {+ Ok(robot) => {+ let allowed = robot.allowed(url.as_str());+ if !allowed {+ match Robot::new("FirecrawlAgent", data.robots_txt.as_bytes()) {+ Ok(robot_alt) => {+ let allowed_alt = robot_alt.allowed(url.as_str());+ if !allowed_alt {+ denial_reasons.insert(link.clone(), "ROBOTS_TXT".to_string());+ return false;+ }+ }+ Err(_) => {+ denial_reasons.insert(link.clone(), "ROBOTS_TXT".to_string());+ return false;+ }+ }+ }+ }+ Err(_) => {+ }}}@@ -513,6 +525,104 @@ mod tests {assert_eq!(result.instructions[0].urls[1], "https://example.com/sitemap2.xml");}+ #[test]+ fn test_filter_links_normal_robots_txt() {+ let data = FilterLinksCall {+ links: vec!["https://example.com/allowed".to_string(), "https://example.com/disallowed".to_string()],+ limit: Some(10),+ includes: vec![],+ excludes: vec![],+ ignore_robots_txt: false,+ robots_txt: "User-agent: *\nDisallow: /disallowed".to_string(),+ max_depth: 10,+ base_url: "https://example.com".to_string(),+ initial_url: "https://example.com".to_string(),+ regex_on_full_url: false,+ allow_backward_crawling: true,+ };++ let result = _filter_links(data).unwrap();+ assert_eq!(result.links.len(), 1);+ assert_eq!(result.links[0], "https://example.com/allowed");+ assert!(result.denial_reasons.contains_key("https://example.com/disallowed"));+ assert_eq!(result.denial_reasons.get("https://example.com/disallowed").unwrap(), "ROBOTS_TXT");+ }++ #[test]+ fn test_filter_links_malformed_robots_txt() {+ let data = FilterLinksCall {+ links: vec!["https://example.com/test".to_string()],+ limit: Some(10),+ includes: vec![],+ excludes: vec![],+ ignore_robots_txt: false,+ robots_txt: "Invalid robots.txt content with \x00 null bytes and malformed syntax".to_string(),+ max_depth: 10,+ base_url: "https://example.com".to_string(),+ initial_url: "https://example.com".to_string(),+ regex_on_full_url: false,+ allow_backward_crawling: true,+ };++ let result = _filter_links(data);+ assert!(result.is_ok());+ let result = result.unwrap();+ assert_eq!(result.links.len(), 1);+ assert_eq!(result.links[0], "https://example.com/test");+ }++ #[test]+ fn test_filter_links_non_utf8_robots_txt() {+ let mut non_utf8_bytes = vec![0xFF, 0xFE];+ non_utf8_bytes.extend_from_slice(b"User-agent: *\nDisallow: /blocked");+ let non_utf8_string = String::from_utf8_lossy(&non_utf8_bytes).to_string();++ let data = FilterLinksCall {+ links: vec!["https://example.com/allowed".to_string()],+ limit: Some(10),+ includes: vec![],+ excludes: vec![],+ ignore_robots_txt: false,+ robots_txt: non_utf8_string,+ max_depth: 10,+ base_url: "https://example.com".to_string(),+ initial_url: "https://example.com".to_string(),+ regex_on_full_url: false,+ allow_backward_crawling: true,+ };++ let result = _filter_links(data);+ assert!(result.is_ok());+ let result = result.unwrap();+ assert_eq!(result.links.len(), 1);+ assert_eq!(result.links[0], "https://example.com/allowed");+ }++ #[test]+ fn test_filter_links_char_boundary_issue() {+ let problematic_content = "User-agent: *\nDisallow: /\u{a0}test";++ let data = FilterLinksCall {+ links: vec!["https://example.com/test".to_string()],+ limit: Some(10),+ includes: vec![],+ excludes: vec![],+ ignore_robots_txt: false,+ robots_txt: problematic_content.to_string(),+ max_depth: 10,+ base_url: "https://example.com".to_string(),+ initial_url: "https://example.com".to_string(),+ regex_on_full_url: false,+ allow_backward_crawling: true,+ };++ let result = _filter_links(data);+ assert!(result.is_ok());+ let result = result.unwrap();+ assert_eq!(result.links.len(), 1);+ assert_eq!(result.links[0], "https://example.com/test");+ }+}/// Frees a string allocated in Rust-land.diff --git a/apps/api/src/__tests__/snips/crawl.test.ts b/apps/api/src/__tests__/snips/crawl.test.tsindex cccbb7370..588259e49 100644--- a/apps/api/src/__tests__/snips/crawl.test.ts+++ b/apps/api/src/__tests__/snips/crawl.test.ts@@ -1,5 +1,6 @@import { asyncCrawl, asyncCrawlWaitForFinish, crawl, crawlOngoing, crawlStart, Identity, idmux, scrapeTimeout } from "./lib";import { describe, it, expect } from "@jest/globals";+import { filterLinks } from "../../lib/crawler";let identity: Identity;@@ -272,3 +273,89 @@ describe("Crawl tests", () => {expect(typeof response.body.id).toBe("string");});});++describe("Robots.txt FFI Integration tests", () => {+ it.concurrent("handles normal robots.txt parsing via FFI", async () => {++ const result = await filterLinks({+ links: ['https://example.com/allowed', 'https://example.com/disallowed'],+ limit: 10,+ max_depth: 10,+ base_url: 'https://example.com',+ initial_url: 'https://example.com',+ regex_on_full_url: false,+ excludes: [],+ includes: [],+ allow_backward_crawling: true,+ ignore_robots_txt: false,+ robots_txt: 'User-agent: *\nDisallow: /disallowed'+ });++ expect(result.links).toHaveLength(1);+ expect(result.links[0]).toBe('https://example.com/allowed');+ expect(result.denial_reasons.has('https://example.com/disallowed')).toBe(true);+ expect(result.denial_reasons.get('https://example.com/disallowed')).toBe('ROBOTS_TXT');+ }, 10000);++ it.concurrent("handles malformed robots.txt without crashing via FFI", async () => {++ const result = await filterLinks({+ links: ['https://example.com/test'],+ limit: 10,+ max_depth: 10,+ base_url: 'https://example.com',+ initial_url: 'https://example.com',+ regex_on_full_url: false,+ excludes: [],+ includes: [],+ allow_backward_crawling: true,+ ignore_robots_txt: false,+ robots_txt: 'Invalid robots.txt content with \x00 null bytes and malformed syntax'+ });++ expect(result.links).toHaveLength(1);+ expect(result.links[0]).toBe('https://example.com/test');+ }, 10000);++ it.concurrent("handles non-UTF8 robots.txt content without crashing via FFI", async () => {++ const nonUtf8Content = String.fromCharCode(0xFF, 0xFE) + 'User-agent: *\nDisallow: /blocked';+ const result = await filterLinks({+ links: ['https://example.com/allowed'],+ limit: 10,+ max_depth: 10,+ base_url: 'https://example.com',+ initial_url: 'https://example.com',+ regex_on_full_url: false,+ excludes: [],+ includes: [],+ allow_backward_crawling: true,+ ignore_robots_txt: false,+ robots_txt: nonUtf8Content+ });++ expect(result.links).toHaveLength(1);+ expect(result.links[0]).toBe('https://example.com/allowed');+ }, 10000);++ it.concurrent("handles char boundary issues without crashing via FFI", async () => {++ const problematicContent = 'User-agent: *\nDisallow: /\u{a0}test';+ const result = await filterLinks({+ links: ['https://example.com/safe'],+ limit: 10,+ max_depth: 10,+ base_url: 'https://example.com',+ initial_url: 'https://example.com',+ regex_on_full_url: false,+ excludes: [],+ includes: [],+ allow_backward_crawling: true,+ ignore_robots_txt: false,+ robots_txt: problematicContent+ });++ expect(result.links).toHaveLength(1);+ expect(result.links[0]).toBe('https://example.com/safe');+ }, 10000);+});