Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Remove .xml.gz from include entries (#2134)
tom1fdfd0d60c23
diff --git a/apps/api/native/src/crawler.rs b/apps/api/native/src/crawler.rsindex 8a4c7aa66..b1f1b8ce7 100644--- a/apps/api/native/src/crawler.rs+++ b/apps/api/native/src/crawler.rs@@ -110,7 +110,8 @@ fn _filter_links(data: FilterLinksCall) -> std::result::Result<FilterLinksResult}let base_url = Url::parse(&data.base_url).map_err(|e| format!("Base URL parse error: {}", e))?;- let initial_url = Url::parse(&data.initial_url).map_err(|e| format!("Initial URL parse error: {}", e))?;+ let initial_url =+ Url::parse(&data.initial_url).map_err(|e| format!("Initial URL parse error: {}", e))?;let excludes_regex = data.excludes@@ -236,7 +237,8 @@ fn _parse_sitemap_xml(xml_content: &str) -> std::result::Result<ParsedSitemap, Sallow_dtd: true,..Default::default()},- ).map_err(|e| format!("XML parsing error: {}", e))?;+ )+ .map_err(|e| format!("XML parsing error: {}", e))?;let root = doc.root_element();match root.tag_name().name() {@@ -303,9 +305,7 @@ pub fn parse_sitemap_xml(xml_content: String) -> Result<ParsedSitemap> {})}-fn _process_sitemap(- xml_content: &str,-) -> std::result::Result<SitemapProcessingResult, String> {+fn _process_sitemap(xml_content: &str) -> std::result::Result<SitemapProcessingResult, String> {let parsed = _parse_sitemap_xml(xml_content)?;let mut instructions = Vec::new();let mut total_count: u32 = 0;@@ -338,7 +338,7 @@ fn _process_sitemap(for url_entry in urlset.url {if !url_entry.loc.is_empty() {let url = url_entry.loc[0].trim();- if url.to_lowercase().ends_with(".xml") {+ if url.to_lowercase().ends_with(".xml") || url.to_lowercase().ends_with(".xml.gz") {xml_sitemaps.push(url.to_string());} else if let Ok(parsed_url) = Url::parse(url) {if !_is_file(&parsed_url) {diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex bdb44a5f7..c58a148c8 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -833,7 +833,8 @@ export class WebCrawler {maxAge?: number,): Promise<number> {const sitemapUrl =- url.endsWith(".xml") || url.endsWith(".xml.gz")+ url.toLowerCase().endsWith(".xml") ||+ url.toLowerCase().endsWith(".xml.gz")? url: `${url}${url.endsWith("/") ? "" : "/"}sitemap.xml`;