Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat: rewrite sitemap XML parsing from JavaScript to Rust (ENG-2904) (#1840)

devin-ai-integration[bot]e0b26f8f3cf5
diff --git a/apps/api/sharedLibs/crawler/Cargo.lock b/apps/api/sharedLibs/crawler/Cargo.lock
index 9b2af422e..3412e19ce 100644
--- a/apps/api/sharedLibs/crawler/Cargo.lock
+++ b/apps/api/sharedLibs/crawler/Cargo.lock
@@ -18,6 +18,7 @@ dependencies = [
"libc",
"regex",
"robotstxt",
+ "roxmltree",
"serde",
"serde_json",
"url",
@@ -242,6 +243,12 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbc52377db80e3fec3a2c748ca603b8b6cacdd34ff89ff4b742a635361d4b4a7"
+[[package]]
+name = "roxmltree"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c20b6793b5c2fa6553b250154b78d6d0db37e72700ae35fad9387a46f487c97"
+
[[package]]
name = "ryu"
version = "1.0.20"
diff --git a/apps/api/sharedLibs/crawler/Cargo.toml b/apps/api/sharedLibs/crawler/Cargo.toml
index cbc4cb9f8..342789129 100644
--- a/apps/api/sharedLibs/crawler/Cargo.toml
+++ b/apps/api/sharedLibs/crawler/Cargo.toml
@@ -10,6 +10,7 @@ serde_json = "1.0"
url = "2.5.4"
regex = "1.11.1"
robotstxt = "0.3.0"
+roxmltree = "0.20"
[lib]
-crate-type = ["cdylib"]
\ No newline at end of file
+crate-type = ["cdylib"]
diff --git a/apps/api/sharedLibs/crawler/src/lib.rs b/apps/api/sharedLibs/crawler/src/lib.rs
index 22f559108..c1f661137 100644
--- a/apps/api/sharedLibs/crawler/src/lib.rs
+++ b/apps/api/sharedLibs/crawler/src/lib.rs
@@ -26,8 +26,52 @@ struct FilterLinksResult {
denial_reasons: HashMap<String, String>,
}
+#[derive(Serialize, Debug)]
+struct SitemapUrl {
+ loc: Vec<String>,
+}
+
+#[derive(Serialize, Debug)]
+struct SitemapEntry {
+ loc: Vec<String>,
+}
+
+#[derive(Serialize, Debug)]
+#[serde(untagged)]
+enum ParsedSitemap {
+ Urlset { urlset: SitemapUrlset },
+ SitemapIndex { sitemapindex: SitemapIndex },
+}
+
+#[derive(Serialize, Debug)]
+struct SitemapUrlset {
+ url: Vec<SitemapUrl>,
+}
+
+#[derive(Serialize, Debug)]
+struct SitemapIndex {
+ sitemap: Vec<SitemapEntry>,
+}
+
+#[derive(Serialize, Debug)]
+struct SitemapInstruction {
+ action: String,
+ urls: Vec<String>,
+ count: usize,
+}
+
+#[derive(Serialize, Debug)]
+struct SitemapProcessingResult {
+ instructions: Vec<SitemapInstruction>,
+ total_count: usize,
+}
+
fn _is_file(url: &Url) -> bool {
let file_extensions = vec![
+ ".png",
+ ".jpg",
+ ".jpeg",
+ ".gif",
".css",
".js",
".ico",
@@ -175,6 +219,298 @@ pub unsafe extern "C" fn filter_links(data: *const libc::c_char) -> *mut libc::c
CString::new(result).unwrap().into_raw()
}
+fn _parse_sitemap_xml(xml_content: &str) -> Result<ParsedSitemap, Box<dyn std::error::Error>> {
+ let doc = roxmltree::Document::parse(xml_content)?;
+ let root = doc.root_element();
+
+ match root.tag_name().name() {
+ "sitemapindex" => {
+ let mut sitemaps = Vec::new();
+
+ for sitemap_node in root.children().filter(|n| n.is_element() && n.tag_name().name() == "sitemap") {
+ if let Some(loc_node) = sitemap_node.children().find(|n| n.is_element() && n.tag_name().name() == "loc") {
+ if let Some(loc_text) = loc_node.text() {
+ sitemaps.push(SitemapEntry {
+ loc: vec![loc_text.to_string()],
+ });
+ }
+ }
+ }
+
+ Ok(ParsedSitemap::SitemapIndex {
+ sitemapindex: SitemapIndex { sitemap: sitemaps }
+ })
+ },
+ "urlset" => {
+ let mut urls = Vec::new();
+
+ for url_node in root.children().filter(|n| n.is_element() && n.tag_name().name() == "url") {
+ if let Some(loc_node) = url_node.children().find(|n| n.is_element() && n.tag_name().name() == "loc") {
+ if let Some(loc_text) = loc_node.text() {
+ urls.push(SitemapUrl {
+ loc: vec![loc_text.to_string()],
+ });
+ }
+ }
+ }
+
+ Ok(ParsedSitemap::Urlset {
+ urlset: SitemapUrlset { url: urls }
+ })
+ },
+ _ => {
+ Err("Invalid sitemap format: root element must be 'sitemapindex' or 'urlset'".into())
+ }
+ }
+}
+
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn parse_sitemap_xml(data: *const libc::c_char) -> *mut libc::c_char {
+ let xml_content = match unsafe { CStr::from_ptr(data).to_str() } {
+ Ok(s) => s,
+ Err(e) => {
+ return CString::new(format!("RUSTFC:ERROR:Failed to parse input data as UTF-8 string: {}", e)).unwrap().into_raw();
+ }
+ };
+
+ let result = match _parse_sitemap_xml(xml_content) {
+ Ok(x) => x,
+ Err(e) => {
+ return CString::new(format!("RUSTFC:ERROR:{}", e)).unwrap().into_raw();
+ }
+ };
+
+ let result = match serde_json::to_string(&result) {
+ Ok(x) => x,
+ Err(e) => {
+ return CString::new(format!("RUSTFC:ERROR:Failed to serialize result as JSON {}", e)).unwrap().into_raw();
+ }
+ };
+
+ CString::new(result).unwrap().into_raw()
+}
+
+fn _process_sitemap(xml_content: &str) -> Result<SitemapProcessingResult, Box<dyn std::error::Error>> {
+ let parsed = _parse_sitemap_xml(xml_content)?;
+ let mut instructions = Vec::new();
+ let mut total_count = 0;
+
+ match parsed {
+ ParsedSitemap::SitemapIndex { sitemapindex } => {
+ let sitemap_urls: Vec<String> = sitemapindex.sitemap
+ .iter()
+ .filter_map(|sitemap| {
+ if !sitemap.loc.is_empty() {
+ Some(sitemap.loc[0].trim().to_string())
+ } else {
+ None
+ }
+ })
+ .collect();
+
+ if !sitemap_urls.is_empty() {
+ instructions.push(SitemapInstruction {
+ action: "recurse".to_string(),
+ urls: sitemap_urls.clone(),
+ count: sitemap_urls.len(),
+ });
+ total_count += sitemap_urls.len();
+ }
+ },
+ ParsedSitemap::Urlset { urlset } => {
+ let mut xml_sitemaps = Vec::new();
+ let mut valid_urls = Vec::new();
+
+ for url_entry in urlset.url {
+ if !url_entry.loc.is_empty() {
+ let url = url_entry.loc[0].trim();
+ if url.to_lowercase().ends_with(".xml") {
+ xml_sitemaps.push(url.to_string());
+ } else if let Ok(parsed_url) = Url::parse(url) {
+ if !_is_file(&parsed_url) {
+ valid_urls.push(url.to_string());
+ }
+ }
+ }
+ }
+
+ if !xml_sitemaps.is_empty() {
+ instructions.push(SitemapInstruction {
+ action: "recurse".to_string(),
+ urls: xml_sitemaps.clone(),
+ count: xml_sitemaps.len(),
+ });
+ total_count += xml_sitemaps.len();
+ }
+
+ if !valid_urls.is_empty() {
+ instructions.push(SitemapInstruction {
+ action: "process".to_string(),
+ urls: valid_urls.clone(),
+ count: valid_urls.len(),
+ });
+ total_count += valid_urls.len();
+ }
+ }
+ }
+
+ Ok(SitemapProcessingResult {
+ instructions,
+ total_count,
+ })
+}
+
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn process_sitemap(data: *const libc::c_char) -> *mut libc::c_char {
+ let xml_content = match unsafe { CStr::from_ptr(data).to_str() } {
+ Ok(s) => s,
+ Err(e) => {
+ return CString::new(format!("RUSTFC:ERROR:Failed to parse input data as UTF-8 string: {}", e)).unwrap().into_raw();
+ }
+ };
+
+ let result = match _process_sitemap(xml_content) {
+ Ok(x) => x,
+ Err(e) => {
+ return CString::new(format!("RUSTFC:ERROR:{}", e)).unwrap().into_raw();
+ }
+ };
+
+ let result = match serde_json::to_string(&result) {
+ Ok(x) => x,
+ Err(e) => {
+ return CString::new(format!("RUSTFC:ERROR:Failed to serialize result as JSON {}", e)).unwrap().into_raw();
+ }
+ };
+
+ CString::new(result).unwrap().into_raw()
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_parse_sitemap_xml_urlset() {
+ let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+ <url>
+ <loc>https://example.com/page1</loc>
+ </url>
+ <url>
+ <loc>https://example.com/page2</loc>
+ </url>
+</urlset>"#;
+
+ let result = _parse_sitemap_xml(xml_content).unwrap();
+ match result {
+ ParsedSitemap::Urlset { urlset } => {
+ assert_eq!(urlset.url.len(), 2);
+ assert_eq!(urlset.url[0].loc[0], "https://example.com/page1");
+ assert_eq!(urlset.url[1].loc[0], "https://example.com/page2");
+ }
+ _ => panic!("Expected Urlset variant"),
+ }
+ }
+
+ #[test]
+ fn test_parse_sitemap_xml_sitemapindex() {
+ let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>
+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+ <sitemap>
+ <loc>https://example.com/sitemap1.xml</loc>
+ </sitemap>
+ <sitemap>
+ <loc>https://example.com/sitemap2.xml</loc>
+ </sitemap>
+</sitemapindex>"#;
+
+ let result = _parse_sitemap_xml(xml_content).unwrap();
+ match result {
+ ParsedSitemap::SitemapIndex { sitemapindex } => {
+ assert_eq!(sitemapindex.sitemap.len(), 2);
+ assert_eq!(sitemapindex.sitemap[0].loc[0], "https://example.com/sitemap1.xml");
+ assert_eq!(sitemapindex.sitemap[1].loc[0], "https://example.com/sitemap2.xml");
+ }
+ _ => panic!("Expected SitemapIndex variant"),
+ }
+ }
+
+ #[test]
+ fn test_parse_sitemap_xml_invalid_root() {
+ let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>
+<invalid xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+ <url>
+ <loc>https://example.com/page1</loc>
+ </url>
+</invalid>"#;
+
+ let result = _parse_sitemap_xml(xml_content);
+ assert!(result.is_err());
+ assert!(result.unwrap_err().to_string().contains("Invalid sitemap format"));
+ }
+
+ #[test]
+ fn test_parse_sitemap_xml_malformed() {
+ let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+ <url>
+ <loc>https://example.com/page1</loc>
+ </url>
+</urlset"#; // Missing closing >
+
+ let result = _parse_sitemap_xml(xml_content);
+ assert!(result.is_err());
+ }
+
+ #[test]
+ fn test_process_sitemap_urlset() {
+ let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+ <url>
+ <loc>https://example.com/page1</loc>
+ </url>
+ <url>
+ <loc>https://example.com/sitemap2.xml</loc>
+ </url>
+ <url>
+ <loc>https://example.com/image.png</loc>
+ </url>
+</urlset>"#;
+
+ let result = _process_sitemap(xml_content).unwrap();
+ assert_eq!(result.instructions.len(), 2);
+
+ let recurse_instruction = result.instructions.iter().find(|i| i.action == "recurse").unwrap();
+ assert_eq!(recurse_instruction.urls.len(), 1);
+ assert_eq!(recurse_instruction.urls[0], "https://example.com/sitemap2.xml");
+
+ let process_instruction = result.instructions.iter().find(|i| i.action == "process").unwrap();
+ assert_eq!(process_instruction.urls.len(), 1);
+ assert_eq!(process_instruction.urls[0], "https://example.com/page1");
+ }
+
+ #[test]
+ fn test_process_sitemap_sitemapindex() {
+ let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>
+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+ <sitemap>
+ <loc>https://example.com/sitemap1.xml</loc>
+ </sitemap>
+ <sitemap>
+ <loc>https://example.com/sitemap2.xml</loc>
+ </sitemap>
+</sitemapindex>"#;
+
+ let result = _process_sitemap(xml_content).unwrap();
+ assert_eq!(result.instructions.len(), 1);
+ assert_eq!(result.instructions[0].action, "recurse");
+ assert_eq!(result.instructions[0].urls.len(), 2);
+ assert_eq!(result.instructions[0].urls[0], "https://example.com/sitemap1.xml");
+ assert_eq!(result.instructions[0].urls[1], "https://example.com/sitemap2.xml");
+ }
+}
+
/// Frees a string allocated in Rust-land.
///
/// # Safety
diff --git a/apps/api/src/lib/crawler.ts b/apps/api/src/lib/crawler.ts
index 82f0bd3d5..862988dce 100644
--- a/apps/api/src/lib/crawler.ts
+++ b/apps/api/src/lib/crawler.ts
@@ -34,6 +34,8 @@ class RustCrawler {
private static instance: RustCrawler;
private _freeString: KoffiFunction;
private _filterLinks: KoffiFunction;
+ private _parseSitemapXml: KoffiFunction;
+ private _processSitemap: KoffiFunction;
private constructor() {
const lib = koffi.load(rustExecutablePath);
@@ -41,6 +43,8 @@ class RustCrawler {
const cstn = "CString:" + crypto.randomUUID();
const freedResultString = koffi.disposable(cstn, "string", this._freeString);
this._filterLinks = lib.func("filter_links", freedResultString, ["string"]);
+ this._parseSitemapXml = lib.func("parse_sitemap_xml", freedResultString, ["string"]);
+ this._processSitemap = lib.func("process_sitemap", freedResultString, ["string"]);
}
public static async getInstance(): Promise<RustCrawler> {
@@ -83,6 +87,56 @@ class RustCrawler {
});
});
}
+
+ public async parseSitemapXml(xmlContent: string): Promise<any> {
+ return new Promise<any>((resolve, reject) => {
+ this._parseSitemapXml.async(xmlContent, (err: Error, res: string) => {
+ if (err) {
+ reject(err);
+ } else {
+ if (res.startsWith("RUSTFC:ERROR:")) {
+ return reject(new Error(res.split("RUSTFC:ERROR:")[1]));
+ }
+
+ if (res === "RUSTFC:ERROR") {
+ return reject(new Error("Something went wrong on the Rust side."));
+ }
+
+ try {
+ const result = JSON.parse(res);
+ resolve(result);
+ } catch (e) {
+ reject(e);
+ }
+ }
+ });
+ });
+ }
+
+ public async processSitemap(xmlContent: string): Promise<any> {
+ return new Promise<any>((resolve, reject) => {
+ this._processSitemap.async(xmlContent, (err: Error, res: string) => {
+ if (err) {
+ reject(err);
+ } else {
+ if (res.startsWith("RUSTFC:ERROR:")) {
+ return reject(new Error(res.split("RUSTFC:ERROR:")[1]));
+ }
+
+ if (res === "RUSTFC:ERROR") {
+ return reject(new Error("Something went wrong on the Rust side."));
+ }
+
+ try {
+ const result = JSON.parse(res);
+ resolve(result);
+ } catch (e) {
+ reject(e);
+ }
+ }
+ });
+ });
+ }
}
export async function filterLinks(
@@ -91,3 +145,13 @@ export async function filterLinks(
const converter = await RustCrawler.getInstance();
return await converter.filterLinks(call);
}
+
+export async function parseSitemapXml(xmlContent: string): Promise<any> {
+ const converter = await RustCrawler.getInstance();
+ return await converter.parseSitemapXml(xmlContent);
+}
+
+export async function processSitemap(xmlContent: string): Promise<any> {
+ const converter = await RustCrawler.getInstance();
+ return await converter.processSitemap(xmlContent);
+}
diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.ts
index 2cec2562d..a35129a82 100644
--- a/apps/api/src/scraper/WebScraper/sitemap.ts
+++ b/apps/api/src/scraper/WebScraper/sitemap.ts
@@ -4,6 +4,7 @@ import { scrapeURL } from "../scrapeURL";
import { scrapeOptions, TimeoutSignal } from "../../controllers/v1/types";
import type { Logger } from "winston";
import { CostTracking } from "../../lib/extract/extraction-service";
+import { parseSitemapXml, processSitemap } from "../../lib/crawler";
const useFireEngine =
process.env.FIRE_ENGINE_BETA_URL !== "" &&
process.env.FIRE_ENGINE_BETA_URL !== undefined;
@@ -94,65 +95,98 @@ export async function getLinksFromSitemap(
}
}
- const parsed = await parseStringPromise(content);
- const root = parsed.urlset || parsed.sitemapindex;
- let count = 0;
+ let instructions;
+ try {
+ instructions = await processSitemap(content);
+ } catch (error) {
+ logger.warn("Rust sitemap processing failed, falling back to JavaScript logic", {
+ method: "getLinksFromSitemap",
+ sitemapUrl,
+ error: error.message,
+ });
+
+ let parsed;
+ try {
+ parsed = await parseSitemapXml(content);
+ } catch (parseError) {
+ parsed = await parseStringPromise(content);
+ }
+
+ const root = parsed.urlset || parsed.sitemapindex;
+ let count = 0;
- if (root && root.sitemap) {
- // Handle sitemap index files
- const sitemapUrls = root.sitemap
- .filter((sitemap) => sitemap.loc && sitemap.loc.length > 0)
- .map((sitemap) => sitemap.loc[0].trim());
+ if (root && root.sitemap) {
+ const sitemapUrls = root.sitemap
+ .filter((sitemap) => sitemap.loc && sitemap.loc.length > 0)
+ .map((sitemap) => sitemap.loc[0].trim());
- const sitemapPromises: Promise<number>[] = sitemapUrls.map((sitemapUrl) =>
- getLinksFromSitemap({ sitemapUrl, urlsHandler, mode, zeroDataRetention }, logger, crawlId, sitemapsHit, abort, mock),
- );
+ const sitemapPromises: Promise<number>[] = sitemapUrls.map((sitemapUrl) =>
+ getLinksFromSitemap({ sitemapUrl, urlsHandler, mode, zeroDataRetention }, logger, crawlId, sitemapsHit, abort, mock),
+ );
- const results = await Promise.all(sitemapPromises);
- count = results.reduce((a, x) => a + x);
- } else if (root && root.url) {
- // Check if any URLs point to additional sitemaps
- const xmlSitemaps: string[] = root.url
- .filter(
- (url) =>
- url.loc &&
- url.loc.length > 0 &&
- url.loc[0].trim().toLowerCase().endsWith(".xml"),
- )
- .map((url) => url.loc[0].trim());
+ const results = await Promise.all(sitemapPromises);
+ count = results.reduce((a, x) => a + x);
+ } else if (root && root.url) {
+ const xmlSitemaps: string[] = root.url
+ .filter(
+ (url) =>
+ url.loc &&
+ url.loc.length > 0 &&
+ url.loc[0].trim().toLowerCase().endsWith(".xml"),
+ )
+ .map((url) => url.loc[0].trim());
- if (xmlSitemaps.length > 0) {
- // Recursively fetch links from additional sitemaps
- const sitemapPromises = xmlSitemaps.map((sitemapUrl) =>
- getLinksFromSitemap(
- { sitemapUrl: sitemapUrl, urlsHandler, mode, zeroDataRetention },
- logger,
- crawlId,
- sitemapsHit,
- abort,
- mock,
- ),
- );
- count += (await Promise.all(sitemapPromises)).reduce(
- (a, x) => a + x,
- 0,
- );
- }
+ if (xmlSitemaps.length > 0) {
+ const sitemapPromises = xmlSitemaps.map((sitemapUrl) =>
+ getLinksFromSitemap(
+ { sitemapUrl: sitemapUrl, urlsHandler, mode, zeroDataRetention },
+ logger,
+ crawlId,
+ sitemapsHit,
+ abort,
+ mock,
+ ),
+ );
+ count += (await Promise.all(sitemapPromises)).reduce(
+ (a, x) => a + x,
+ 0,
+ );
+ }
- const validUrls = root.url
- .filter(
- (url) =>
- url.loc &&
- url.loc.length > 0 &&
- !url.loc[0].trim().toLowerCase().endsWith(".xml") &&
- !WebCrawler.prototype.isFile(url.loc[0].trim()),
- )
- .map((url) => url.loc[0].trim());
- count += validUrls.length;
+ const validUrls = root.url
+ .filter(
+ (url) =>
+ url.loc &&
+ url.loc.length > 0 &&
+ !url.loc[0].trim().toLowerCase().endsWith(".xml") &&
+ !WebCrawler.prototype.isFile(url.loc[0].trim()),
+ )
+ .map((url) => url.loc[0].trim());
+ count += validUrls.length;
- const h = urlsHandler(validUrls);
- if (h instanceof Promise) {
- await h;
+ const h = urlsHandler(validUrls);
+ if (h instanceof Promise) {
+ await h;
+ }
+ }
+
+ return count;
+ }
+
+ let count = 0;
+ for (const instruction of instructions.instructions) {
+ if (instruction.action === "recurse") {
+ const sitemapPromises: Promise<number>[] = instruction.urls.map((sitemapUrl) =>
+ getLinksFromSitemap({ sitemapUrl, urlsHandler, mode, zeroDataRetention }, logger, crawlId, sitemapsHit, abort, mock),
+ );
+ const results = await Promise.all(sitemapPromises);
+ count += results.reduce((a, x) => a + x);
+ } else if (instruction.action === "process") {
+ count += instruction.urls.length;
+ const h = urlsHandler(instruction.urls);
+ if (h instanceof Promise) {
+ await h;
+ }
}
}