Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(crawler): handle negative limit
Gergő Móricz7336e84f961f
diff --git a/apps/api/sharedLibs/crawler/src/lib.rs b/apps/api/sharedLibs/crawler/src/lib.rsindex ed045470b..339b87ad4 100644--- a/apps/api/sharedLibs/crawler/src/lib.rs+++ b/apps/api/sharedLibs/crawler/src/lib.rs@@ -8,7 +8,7 @@ use robotstxt::DefaultMatcher;#[derive(Deserialize)]struct FilterLinksCall {links: Vec<String>,- limit: Option<u32>,+ limit: Option<i64>,max_depth: u32,base_url: String,initial_url: String,@@ -59,6 +59,18 @@ fn _get_url_depth(path: &str) -> u32 {fn _filter_links(data: FilterLinksCall) -> Result<FilterLinksResult, Box<dyn std::error::Error>> {let mut denial_reasons = HashMap::new();++ let limit = data.limit.and_then(|x| if x < 0 { Some(0) } else { Some(x as usize) });++ let limit = if let Some(limit) = limit {+ limit+ } else {+ return Ok(FilterLinksResult {+ links: Vec::with_capacity(0),+ denial_reasons,+ });+ };+let base_url = Url::parse(&data.base_url)?;let initial_url = Url::parse(&data.initial_url)?;@@ -127,13 +139,9 @@ fn _filter_links(data: FilterLinksCall) -> Result<FilterLinksResult, Box<dyn std}true- });-- let links = if let Some(limit) = data.limit {- links.take(limit as usize).collect::<Vec<_>>()- } else {- links.collect::<Vec<_>>()- };+ })+ .take(limit)+ .collect::<Vec<_>>();Ok(FilterLinksResult {links,