Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(crawler/sitemap): improvements (#1842)

Gergő Móriczebdd2ea93b37
diff --git a/apps/api/sharedLibs/crawler/src/lib.rs b/apps/api/sharedLibs/crawler/src/lib.rs
index 45d70240e..aec7935a7 100644
--- a/apps/api/sharedLibs/crawler/src/lib.rs
+++ b/apps/api/sharedLibs/crawler/src/lib.rs
@@ -220,7 +220,10 @@ pub unsafe extern "C" fn filter_links(data: *const libc::c_char) -> *mut libc::c
}
fn _parse_sitemap_xml(xml_content: &str) -> Result<ParsedSitemap, Box<dyn std::error::Error>> {
- let doc = roxmltree::Document::parse(xml_content)?;
+ let doc = roxmltree::Document::parse_with_options(
+ xml_content,
+ roxmltree::ParsingOptions { allow_dtd: true, ..Default::default() },
+ )?;
let root = doc.root_element();
match root.tag_name().name() {
diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.ts
index a35129a82..30981f8a1 100644
--- a/apps/api/src/scraper/WebScraper/sitemap.ts
+++ b/apps/api/src/scraper/WebScraper/sitemap.ts
@@ -109,6 +109,11 @@ export async function getLinksFromSitemap(
try {
parsed = await parseSitemapXml(content);
} catch (parseError) {
+ logger.warn("Rust XML parsing failed, falling back to JavaScript logic", {
+ method: "getLinksFromSitemap",
+ sitemapUrl,
+ error: parseError.message,
+ });
parsed = await parseStringPromise(content);
}