Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: rewrite sitemap XML parsing from JavaScript to Rust (ENG-2904) (#1840)
devin-ai-integration[bot]e0b26f8f3cf5
diff --git a/apps/api/sharedLibs/crawler/Cargo.lock b/apps/api/sharedLibs/crawler/Cargo.lockindex 9b2af422e..3412e19ce 100644--- a/apps/api/sharedLibs/crawler/Cargo.lock+++ b/apps/api/sharedLibs/crawler/Cargo.lock@@ -18,6 +18,7 @@ dependencies = ["libc","regex","robotstxt",+ "roxmltree","serde","serde_json","url",@@ -242,6 +243,12 @@ version = "0.3.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "bbc52377db80e3fec3a2c748ca603b8b6cacdd34ff89ff4b742a635361d4b4a7"+[[package]]+name = "roxmltree"+version = "0.20.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6c20b6793b5c2fa6553b250154b78d6d0db37e72700ae35fad9387a46f487c97"+[[package]]name = "ryu"version = "1.0.20"diff --git a/apps/api/sharedLibs/crawler/Cargo.toml b/apps/api/sharedLibs/crawler/Cargo.tomlindex cbc4cb9f8..342789129 100644--- a/apps/api/sharedLibs/crawler/Cargo.toml+++ b/apps/api/sharedLibs/crawler/Cargo.toml@@ -10,6 +10,7 @@ serde_json = "1.0"url = "2.5.4"regex = "1.11.1"robotstxt = "0.3.0"+roxmltree = "0.20"[lib]-crate-type = ["cdylib"]\ No newline at end of file+crate-type = ["cdylib"]diff --git a/apps/api/sharedLibs/crawler/src/lib.rs b/apps/api/sharedLibs/crawler/src/lib.rsindex 22f559108..c1f661137 100644--- a/apps/api/sharedLibs/crawler/src/lib.rs+++ b/apps/api/sharedLibs/crawler/src/lib.rs@@ -26,8 +26,52 @@ struct FilterLinksResult {denial_reasons: HashMap<String, String>,}+#[derive(Serialize, Debug)]+struct SitemapUrl {+ loc: Vec<String>,+}++#[derive(Serialize, Debug)]+struct SitemapEntry {+ loc: Vec<String>,+}++#[derive(Serialize, Debug)]+#[serde(untagged)]+enum ParsedSitemap {+ Urlset { urlset: SitemapUrlset },+ SitemapIndex { sitemapindex: SitemapIndex },+}++#[derive(Serialize, Debug)]+struct SitemapUrlset {+ url: Vec<SitemapUrl>,+}++#[derive(Serialize, Debug)]+struct SitemapIndex {+ sitemap: Vec<SitemapEntry>,+}++#[derive(Serialize, Debug)]+struct SitemapInstruction {+ action: String,+ urls: Vec<String>,+ count: usize,+}++#[derive(Serialize, Debug)]+struct SitemapProcessingResult {+ instructions: Vec<SitemapInstruction>,+ total_count: usize,+}+fn _is_file(url: &Url) -> bool {let file_extensions = vec![+ ".png",+ ".jpg",+ ".jpeg",+ ".gif",".css",".js",".ico",@@ -175,6 +219,298 @@ pub unsafe extern "C" fn filter_links(data: *const libc::c_char) -> *mut libc::cCString::new(result).unwrap().into_raw()}+fn _parse_sitemap_xml(xml_content: &str) -> Result<ParsedSitemap, Box<dyn std::error::Error>> {+ let doc = roxmltree::Document::parse(xml_content)?;+ let root = doc.root_element();++ match root.tag_name().name() {+ "sitemapindex" => {+ let mut sitemaps = Vec::new();++ for sitemap_node in root.children().filter(|n| n.is_element() && n.tag_name().name() == "sitemap") {+ if let Some(loc_node) = sitemap_node.children().find(|n| n.is_element() && n.tag_name().name() == "loc") {+ if let Some(loc_text) = loc_node.text() {+ sitemaps.push(SitemapEntry {+ loc: vec![loc_text.to_string()],+ });+ }+ }+ }++ Ok(ParsedSitemap::SitemapIndex {+ sitemapindex: SitemapIndex { sitemap: sitemaps }+ })+ },+ "urlset" => {+ let mut urls = Vec::new();++ for url_node in root.children().filter(|n| n.is_element() && n.tag_name().name() == "url") {+ if let Some(loc_node) = url_node.children().find(|n| n.is_element() && n.tag_name().name() == "loc") {+ if let Some(loc_text) = loc_node.text() {+ urls.push(SitemapUrl {+ loc: vec![loc_text.to_string()],+ });+ }+ }+ }++ Ok(ParsedSitemap::Urlset {+ urlset: SitemapUrlset { url: urls }+ })+ },+ _ => {+ Err("Invalid sitemap format: root element must be 'sitemapindex' or 'urlset'".into())+ }+ }+}++#[unsafe(no_mangle)]+pub unsafe extern "C" fn parse_sitemap_xml(data: *const libc::c_char) -> *mut libc::c_char {+ let xml_content = match unsafe { CStr::from_ptr(data).to_str() } {+ Ok(s) => s,+ Err(e) => {+ return CString::new(format!("RUSTFC:ERROR:Failed to parse input data as UTF-8 string: {}", e)).unwrap().into_raw();+ }+ };++ let result = match _parse_sitemap_xml(xml_content) {+ Ok(x) => x,+ Err(e) => {+ return CString::new(format!("RUSTFC:ERROR:{}", e)).unwrap().into_raw();+ }+ };++ let result = match serde_json::to_string(&result) {+ Ok(x) => x,+ Err(e) => {+ return CString::new(format!("RUSTFC:ERROR:Failed to serialize result as JSON {}", e)).unwrap().into_raw();+ }+ };++ CString::new(result).unwrap().into_raw()+}++fn _process_sitemap(xml_content: &str) -> Result<SitemapProcessingResult, Box<dyn std::error::Error>> {+ let parsed = _parse_sitemap_xml(xml_content)?;+ let mut instructions = Vec::new();+ let mut total_count = 0;++ match parsed {+ ParsedSitemap::SitemapIndex { sitemapindex } => {+ let sitemap_urls: Vec<String> = sitemapindex.sitemap+ .iter()+ .filter_map(|sitemap| {+ if !sitemap.loc.is_empty() {+ Some(sitemap.loc[0].trim().to_string())+ } else {+ None+ }+ })+ .collect();++ if !sitemap_urls.is_empty() {+ instructions.push(SitemapInstruction {+ action: "recurse".to_string(),+ urls: sitemap_urls.clone(),+ count: sitemap_urls.len(),+ });+ total_count += sitemap_urls.len();+ }+ },+ ParsedSitemap::Urlset { urlset } => {+ let mut xml_sitemaps = Vec::new();+ let mut valid_urls = Vec::new();++ for url_entry in urlset.url {+ if !url_entry.loc.is_empty() {+ let url = url_entry.loc[0].trim();+ if url.to_lowercase().ends_with(".xml") {+ xml_sitemaps.push(url.to_string());+ } else if let Ok(parsed_url) = Url::parse(url) {+ if !_is_file(&parsed_url) {+ valid_urls.push(url.to_string());+ }+ }+ }+ }++ if !xml_sitemaps.is_empty() {+ instructions.push(SitemapInstruction {+ action: "recurse".to_string(),+ urls: xml_sitemaps.clone(),+ count: xml_sitemaps.len(),+ });+ total_count += xml_sitemaps.len();+ }++ if !valid_urls.is_empty() {+ instructions.push(SitemapInstruction {+ action: "process".to_string(),+ urls: valid_urls.clone(),+ count: valid_urls.len(),+ });+ total_count += valid_urls.len();+ }+ }+ }++ Ok(SitemapProcessingResult {+ instructions,+ total_count,+ })+}++#[unsafe(no_mangle)]+pub unsafe extern "C" fn process_sitemap(data: *const libc::c_char) -> *mut libc::c_char {+ let xml_content = match unsafe { CStr::from_ptr(data).to_str() } {+ Ok(s) => s,+ Err(e) => {+ return CString::new(format!("RUSTFC:ERROR:Failed to parse input data as UTF-8 string: {}", e)).unwrap().into_raw();+ }+ };++ let result = match _process_sitemap(xml_content) {+ Ok(x) => x,+ Err(e) => {+ return CString::new(format!("RUSTFC:ERROR:{}", e)).unwrap().into_raw();+ }+ };++ let result = match serde_json::to_string(&result) {+ Ok(x) => x,+ Err(e) => {+ return CString::new(format!("RUSTFC:ERROR:Failed to serialize result as JSON {}", e)).unwrap().into_raw();+ }+ };++ CString::new(result).unwrap().into_raw()+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn test_parse_sitemap_xml_urlset() {+ let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">+ <url>+ <loc>https://example.com/page1</loc>+ </url>+ <url>+ <loc>https://example.com/page2</loc>+ </url>+</urlset>"#;++ let result = _parse_sitemap_xml(xml_content).unwrap();+ match result {+ ParsedSitemap::Urlset { urlset } => {+ assert_eq!(urlset.url.len(), 2);+ assert_eq!(urlset.url[0].loc[0], "https://example.com/page1");+ assert_eq!(urlset.url[1].loc[0], "https://example.com/page2");+ }+ _ => panic!("Expected Urlset variant"),+ }+ }++ #[test]+ fn test_parse_sitemap_xml_sitemapindex() {+ let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">+ <sitemap>+ <loc>https://example.com/sitemap1.xml</loc>+ </sitemap>+ <sitemap>+ <loc>https://example.com/sitemap2.xml</loc>+ </sitemap>+</sitemapindex>"#;++ let result = _parse_sitemap_xml(xml_content).unwrap();+ match result {+ ParsedSitemap::SitemapIndex { sitemapindex } => {+ assert_eq!(sitemapindex.sitemap.len(), 2);+ assert_eq!(sitemapindex.sitemap[0].loc[0], "https://example.com/sitemap1.xml");+ assert_eq!(sitemapindex.sitemap[1].loc[0], "https://example.com/sitemap2.xml");+ }+ _ => panic!("Expected SitemapIndex variant"),+ }+ }++ #[test]+ fn test_parse_sitemap_xml_invalid_root() {+ let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>+<invalid xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">+ <url>+ <loc>https://example.com/page1</loc>+ </url>+</invalid>"#;++ let result = _parse_sitemap_xml(xml_content);+ assert!(result.is_err());+ assert!(result.unwrap_err().to_string().contains("Invalid sitemap format"));+ }++ #[test]+ fn test_parse_sitemap_xml_malformed() {+ let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">+ <url>+ <loc>https://example.com/page1</loc>+ </url>+</urlset"#; // Missing closing >++ let result = _parse_sitemap_xml(xml_content);+ assert!(result.is_err());+ }++ #[test]+ fn test_process_sitemap_urlset() {+ let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">+ <url>+ <loc>https://example.com/page1</loc>+ </url>+ <url>+ <loc>https://example.com/sitemap2.xml</loc>+ </url>+ <url>+ <loc>https://example.com/image.png</loc>+ </url>+</urlset>"#;++ let result = _process_sitemap(xml_content).unwrap();+ assert_eq!(result.instructions.len(), 2);++ let recurse_instruction = result.instructions.iter().find(|i| i.action == "recurse").unwrap();+ assert_eq!(recurse_instruction.urls.len(), 1);+ assert_eq!(recurse_instruction.urls[0], "https://example.com/sitemap2.xml");++ let process_instruction = result.instructions.iter().find(|i| i.action == "process").unwrap();+ assert_eq!(process_instruction.urls.len(), 1);+ assert_eq!(process_instruction.urls[0], "https://example.com/page1");+ }++ #[test]+ fn test_process_sitemap_sitemapindex() {+ let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">+ <sitemap>+ <loc>https://example.com/sitemap1.xml</loc>+ </sitemap>+ <sitemap>+ <loc>https://example.com/sitemap2.xml</loc>+ </sitemap>+</sitemapindex>"#;++ let result = _process_sitemap(xml_content).unwrap();+ assert_eq!(result.instructions.len(), 1);+ assert_eq!(result.instructions[0].action, "recurse");+ assert_eq!(result.instructions[0].urls.len(), 2);+ assert_eq!(result.instructions[0].urls[0], "https://example.com/sitemap1.xml");+ assert_eq!(result.instructions[0].urls[1], "https://example.com/sitemap2.xml");+ }+}+/// Frees a string allocated in Rust-land.////// # Safetydiff --git a/apps/api/src/lib/crawler.ts b/apps/api/src/lib/crawler.tsindex 82f0bd3d5..862988dce 100644--- a/apps/api/src/lib/crawler.ts+++ b/apps/api/src/lib/crawler.ts@@ -34,6 +34,8 @@ class RustCrawler {private static instance: RustCrawler;private _freeString: KoffiFunction;private _filterLinks: KoffiFunction;+ private _parseSitemapXml: KoffiFunction;+ private _processSitemap: KoffiFunction;private constructor() {const lib = koffi.load(rustExecutablePath);@@ -41,6 +43,8 @@ class RustCrawler {const cstn = "CString:" + crypto.randomUUID();const freedResultString = koffi.disposable(cstn, "string", this._freeString);this._filterLinks = lib.func("filter_links", freedResultString, ["string"]);+ this._parseSitemapXml = lib.func("parse_sitemap_xml", freedResultString, ["string"]);+ this._processSitemap = lib.func("process_sitemap", freedResultString, ["string"]);}public static async getInstance(): Promise<RustCrawler> {@@ -83,6 +87,56 @@ class RustCrawler {});});}++ public async parseSitemapXml(xmlContent: string): Promise<any> {+ return new Promise<any>((resolve, reject) => {+ this._parseSitemapXml.async(xmlContent, (err: Error, res: string) => {+ if (err) {+ reject(err);+ } else {+ if (res.startsWith("RUSTFC:ERROR:")) {+ return reject(new Error(res.split("RUSTFC:ERROR:")[1]));+ }++ if (res === "RUSTFC:ERROR") {+ return reject(new Error("Something went wrong on the Rust side."));+ }++ try {+ const result = JSON.parse(res);+ resolve(result);+ } catch (e) {+ reject(e);+ }+ }+ });+ });+ }++ public async processSitemap(xmlContent: string): Promise<any> {+ return new Promise<any>((resolve, reject) => {+ this._processSitemap.async(xmlContent, (err: Error, res: string) => {+ if (err) {+ reject(err);+ } else {+ if (res.startsWith("RUSTFC:ERROR:")) {+ return reject(new Error(res.split("RUSTFC:ERROR:")[1]));+ }++ if (res === "RUSTFC:ERROR") {+ return reject(new Error("Something went wrong on the Rust side."));+ }++ try {+ const result = JSON.parse(res);+ resolve(result);+ } catch (e) {+ reject(e);+ }+ }+ });+ });+ }}export async function filterLinks(@@ -91,3 +145,13 @@ export async function filterLinks(const converter = await RustCrawler.getInstance();return await converter.filterLinks(call);}++export async function parseSitemapXml(xmlContent: string): Promise<any> {+ const converter = await RustCrawler.getInstance();+ return await converter.parseSitemapXml(xmlContent);+}++export async function processSitemap(xmlContent: string): Promise<any> {+ const converter = await RustCrawler.getInstance();+ return await converter.processSitemap(xmlContent);+}diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.tsindex 2cec2562d..a35129a82 100644--- a/apps/api/src/scraper/WebScraper/sitemap.ts+++ b/apps/api/src/scraper/WebScraper/sitemap.ts@@ -4,6 +4,7 @@ import { scrapeURL } from "../scrapeURL";import { scrapeOptions, TimeoutSignal } from "../../controllers/v1/types";import type { Logger } from "winston";import { CostTracking } from "../../lib/extract/extraction-service";+import { parseSitemapXml, processSitemap } from "../../lib/crawler";const useFireEngine =process.env.FIRE_ENGINE_BETA_URL !== "" &&process.env.FIRE_ENGINE_BETA_URL !== undefined;@@ -94,65 +95,98 @@ export async function getLinksFromSitemap(}}- const parsed = await parseStringPromise(content);- const root = parsed.urlset || parsed.sitemapindex;- let count = 0;+ let instructions;+ try {+ instructions = await processSitemap(content);+ } catch (error) {+ logger.warn("Rust sitemap processing failed, falling back to JavaScript logic", {+ method: "getLinksFromSitemap",+ sitemapUrl,+ error: error.message,+ });++ let parsed;+ try {+ parsed = await parseSitemapXml(content);+ } catch (parseError) {+ parsed = await parseStringPromise(content);+ }++ const root = parsed.urlset || parsed.sitemapindex;+ let count = 0;- if (root && root.sitemap) {- // Handle sitemap index files- const sitemapUrls = root.sitemap- .filter((sitemap) => sitemap.loc && sitemap.loc.length > 0)- .map((sitemap) => sitemap.loc[0].trim());+ if (root && root.sitemap) {+ const sitemapUrls = root.sitemap+ .filter((sitemap) => sitemap.loc && sitemap.loc.length > 0)+ .map((sitemap) => sitemap.loc[0].trim());- const sitemapPromises: Promise<number>[] = sitemapUrls.map((sitemapUrl) =>- getLinksFromSitemap({ sitemapUrl, urlsHandler, mode, zeroDataRetention }, logger, crawlId, sitemapsHit, abort, mock),- );+ const sitemapPromises: Promise<number>[] = sitemapUrls.map((sitemapUrl) =>+ getLinksFromSitemap({ sitemapUrl, urlsHandler, mode, zeroDataRetention }, logger, crawlId, sitemapsHit, abort, mock),+ );- const results = await Promise.all(sitemapPromises);- count = results.reduce((a, x) => a + x);- } else if (root && root.url) {- // Check if any URLs point to additional sitemaps- const xmlSitemaps: string[] = root.url- .filter(- (url) =>- url.loc &&- url.loc.length > 0 &&- url.loc[0].trim().toLowerCase().endsWith(".xml"),- )- .map((url) => url.loc[0].trim());+ const results = await Promise.all(sitemapPromises);+ count = results.reduce((a, x) => a + x);+ } else if (root && root.url) {+ const xmlSitemaps: string[] = root.url+ .filter(+ (url) =>+ url.loc &&+ url.loc.length > 0 &&+ url.loc[0].trim().toLowerCase().endsWith(".xml"),+ )+ .map((url) => url.loc[0].trim());- if (xmlSitemaps.length > 0) {- // Recursively fetch links from additional sitemaps- const sitemapPromises = xmlSitemaps.map((sitemapUrl) =>- getLinksFromSitemap(- { sitemapUrl: sitemapUrl, urlsHandler, mode, zeroDataRetention },- logger,- crawlId,- sitemapsHit,- abort,- mock,- ),- );- count += (await Promise.all(sitemapPromises)).reduce(- (a, x) => a + x,- 0,- );- }+ if (xmlSitemaps.length > 0) {+ const sitemapPromises = xmlSitemaps.map((sitemapUrl) =>+ getLinksFromSitemap(+ { sitemapUrl: sitemapUrl, urlsHandler, mode, zeroDataRetention },+ logger,+ crawlId,+ sitemapsHit,+ abort,+ mock,+ ),+ );+ count += (await Promise.all(sitemapPromises)).reduce(+ (a, x) => a + x,+ 0,+ );+ }- const validUrls = root.url- .filter(- (url) =>- url.loc &&- url.loc.length > 0 &&- !url.loc[0].trim().toLowerCase().endsWith(".xml") &&- !WebCrawler.prototype.isFile(url.loc[0].trim()),- )- .map((url) => url.loc[0].trim());- count += validUrls.length;+ const validUrls = root.url+ .filter(+ (url) =>+ url.loc &&+ url.loc.length > 0 &&+ !url.loc[0].trim().toLowerCase().endsWith(".xml") &&+ !WebCrawler.prototype.isFile(url.loc[0].trim()),+ )+ .map((url) => url.loc[0].trim());+ count += validUrls.length;- const h = urlsHandler(validUrls);- if (h instanceof Promise) {- await h;+ const h = urlsHandler(validUrls);+ if (h instanceof Promise) {+ await h;+ }+ }++ return count;+ }++ let count = 0;+ for (const instruction of instructions.instructions) {+ if (instruction.action === "recurse") {+ const sitemapPromises: Promise<number>[] = instruction.urls.map((sitemapUrl) =>+ getLinksFromSitemap({ sitemapUrl, urlsHandler, mode, zeroDataRetention }, logger, crawlId, sitemapsHit, abort, mock),+ );+ const results = await Promise.all(sitemapPromises);+ count += results.reduce((a, x) => a + x);+ } else if (instruction.action === "process") {+ count += instruction.urls.length;+ const h = urlsHandler(instruction.urls);+ if (h instanceof Promise) {+ await h;+ }}}