Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(crawler): handle infinite limit
Gergő Móricz7695a8c77677
diff --git a/apps/api/sharedLibs/crawler/src/lib.rs b/apps/api/sharedLibs/crawler/src/lib.rsindex 5989309e3..ed045470b 100644--- a/apps/api/sharedLibs/crawler/src/lib.rs+++ b/apps/api/sharedLibs/crawler/src/lib.rs@@ -8,7 +8,7 @@ use robotstxt::DefaultMatcher;#[derive(Deserialize)]struct FilterLinksCall {links: Vec<String>,- limit: u32,+ limit: Option<u32>,max_depth: u32,base_url: String,initial_url: String,@@ -127,9 +127,13 @@ fn _filter_links(data: FilterLinksCall) -> Result<FilterLinksResult, Box<dyn std}true- })- .take(data.limit as usize)- .collect();+ });++ let links = if let Some(limit) = data.limit {+ links.take(limit as usize).collect::<Vec<_>>()+ } else {+ links.collect::<Vec<_>>()+ };Ok(FilterLinksResult {links,diff --git a/apps/api/src/lib/crawler.ts b/apps/api/src/lib/crawler.tsindex a0a93dc42..eb281235b 100644--- a/apps/api/src/lib/crawler.ts+++ b/apps/api/src/lib/crawler.ts@@ -13,7 +13,7 @@ const rustExecutablePath = join(export type FilterLinksCall = {links: string[],- limit: number,+ limit: number | undefined,max_depth: number,base_url: string,initial_url: string,@@ -88,6 +88,7 @@ class RustCrawler {export async function filterLinks(call: FilterLinksCall,): Promise<FilterLinksResult> {+ console.log(call);const converter = await RustCrawler.getInstance();return await converter.filterLinks(call);}diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex d5b52f8ec..67ff04442 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -147,7 +147,7 @@ export class WebCrawler {try {const res = await filterLinks({links: sitemapLinks,- limit,+ limit: isFinite(limit) ? limit : undefined,max_depth: maxDepth,base_url: this.baseUrl,initial_url: this.initialUrl,