Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(crawler): handle infinite limit

Gergő Móricz7695a8c77677
diff --git a/apps/api/sharedLibs/crawler/src/lib.rs b/apps/api/sharedLibs/crawler/src/lib.rs
index 5989309e3..ed045470b 100644
--- a/apps/api/sharedLibs/crawler/src/lib.rs
+++ b/apps/api/sharedLibs/crawler/src/lib.rs
@@ -8,7 +8,7 @@ use robotstxt::DefaultMatcher;
#[derive(Deserialize)]
struct FilterLinksCall {
links: Vec<String>,
- limit: u32,
+ limit: Option<u32>,
max_depth: u32,
base_url: String,
initial_url: String,
@@ -127,9 +127,13 @@ fn _filter_links(data: FilterLinksCall) -> Result<FilterLinksResult, Box<dyn std
}
true
- })
- .take(data.limit as usize)
- .collect();
+ });
+
+ let links = if let Some(limit) = data.limit {
+ links.take(limit as usize).collect::<Vec<_>>()
+ } else {
+ links.collect::<Vec<_>>()
+ };
Ok(FilterLinksResult {
links,
diff --git a/apps/api/src/lib/crawler.ts b/apps/api/src/lib/crawler.ts
index a0a93dc42..eb281235b 100644
--- a/apps/api/src/lib/crawler.ts
+++ b/apps/api/src/lib/crawler.ts
@@ -13,7 +13,7 @@ const rustExecutablePath = join(
export type FilterLinksCall = {
links: string[],
- limit: number,
+ limit: number | undefined,
max_depth: number,
base_url: string,
initial_url: string,
@@ -88,6 +88,7 @@ class RustCrawler {
export async function filterLinks(
call: FilterLinksCall,
): Promise<FilterLinksResult> {
+ console.log(call);
const converter = await RustCrawler.getInstance();
return await converter.filterLinks(call);
}
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index d5b52f8ec..67ff04442 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -147,7 +147,7 @@ export class WebCrawler {
try {
const res = await filterLinks({
links: sitemapLinks,
- limit,
+ limit: isFinite(limit) ? limit : undefined,
max_depth: maxDepth,
base_url: this.baseUrl,
initial_url: this.initialUrl,