Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Remove .xml.gz from include entries (#2134)

tom1fdfd0d60c23
diff --git a/apps/api/native/src/crawler.rs b/apps/api/native/src/crawler.rs
index 8a4c7aa66..b1f1b8ce7 100644
--- a/apps/api/native/src/crawler.rs
+++ b/apps/api/native/src/crawler.rs
@@ -110,7 +110,8 @@ fn _filter_links(data: FilterLinksCall) -> std::result::Result<FilterLinksResult
}
let base_url = Url::parse(&data.base_url).map_err(|e| format!("Base URL parse error: {}", e))?;
- let initial_url = Url::parse(&data.initial_url).map_err(|e| format!("Initial URL parse error: {}", e))?;
+ let initial_url =
+ Url::parse(&data.initial_url).map_err(|e| format!("Initial URL parse error: {}", e))?;
let excludes_regex = data
.excludes
@@ -236,7 +237,8 @@ fn _parse_sitemap_xml(xml_content: &str) -> std::result::Result<ParsedSitemap, S
allow_dtd: true,
..Default::default()
},
- ).map_err(|e| format!("XML parsing error: {}", e))?;
+ )
+ .map_err(|e| format!("XML parsing error: {}", e))?;
let root = doc.root_element();
match root.tag_name().name() {
@@ -303,9 +305,7 @@ pub fn parse_sitemap_xml(xml_content: String) -> Result<ParsedSitemap> {
})
}
-fn _process_sitemap(
- xml_content: &str,
-) -> std::result::Result<SitemapProcessingResult, String> {
+fn _process_sitemap(xml_content: &str) -> std::result::Result<SitemapProcessingResult, String> {
let parsed = _parse_sitemap_xml(xml_content)?;
let mut instructions = Vec::new();
let mut total_count: u32 = 0;
@@ -338,7 +338,7 @@ fn _process_sitemap(
for url_entry in urlset.url {
if !url_entry.loc.is_empty() {
let url = url_entry.loc[0].trim();
- if url.to_lowercase().ends_with(".xml") {
+ if url.to_lowercase().ends_with(".xml") || url.to_lowercase().ends_with(".xml.gz") {
xml_sitemaps.push(url.to_string());
} else if let Ok(parsed_url) = Url::parse(url) {
if !_is_file(&parsed_url) {
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index bdb44a5f7..c58a148c8 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -833,7 +833,8 @@ export class WebCrawler {
maxAge?: number,
): Promise<number> {
const sitemapUrl =
- url.endsWith(".xml") || url.endsWith(".xml.gz")
+ url.toLowerCase().endsWith(".xml") ||
+ url.toLowerCase().endsWith(".xml.gz")
? url
: `${url}${url.endsWith("/") ? "" : "/"}sitemap.xml`;