Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Restore Rust link filtering logic (#1822)

Gergő Móricz17241e098349
diff --git a/apps/api/sharedLibs/crawler/src/lib.rs b/apps/api/sharedLibs/crawler/src/lib.rs
index 339b87ad4..22f559108 100644
--- a/apps/api/sharedLibs/crawler/src/lib.rs
+++ b/apps/api/sharedLibs/crawler/src/lib.rs
@@ -1,4 +1,4 @@
-use std::collections::HashMap;
+use std::{collections::HashMap, usize};
use serde::{Deserialize, Serialize};
use std::ffi::{CStr, CString};
use url::Url;
@@ -60,16 +60,16 @@ fn _get_url_depth(path: &str) -> u32 {
fn _filter_links(data: FilterLinksCall) -> Result<FilterLinksResult, Box<dyn std::error::Error>> {
let mut denial_reasons = HashMap::new();
- let limit = data.limit.and_then(|x| if x < 0 { Some(0) } else { Some(x as usize) });
+ let limit = data.limit
+ .and_then(|x| if x < 0 { Some(0) } else { Some(x as usize) })
+ .unwrap_or(usize::MAX);
- let limit = if let Some(limit) = limit {
- limit
- } else {
+ if limit == 0 {
return Ok(FilterLinksResult {
links: Vec::with_capacity(0),
denial_reasons,
});
- };
+ }
let base_url = Url::parse(&data.base_url)?;
let initial_url = Url::parse(&data.initial_url)?;
diff --git a/apps/api/src/__tests__/snips/crawl.test.ts b/apps/api/src/__tests__/snips/crawl.test.ts
index 907018350..93bd96d04 100644
--- a/apps/api/src/__tests__/snips/crawl.test.ts
+++ b/apps/api/src/__tests__/snips/crawl.test.ts
@@ -13,10 +13,22 @@ beforeAll(async () => {
describe("Crawl tests", () => {
it.concurrent("works", async () => {
- await crawl({
+ const results = await crawl({
+ url: "https://firecrawl.dev",
+ limit: 10,
+ }, identity);
+
+ expect(results.completed).toBe(10);
+ }, 10 * scrapeTimeout);
+
+ it.concurrent("works with ignoreSitemap: true", async () => {
+ const results = await crawl({
url: "https://firecrawl.dev",
limit: 10,
+ ignoreSitemap: true,
}, identity);
+
+ expect(results.completed).toBe(10);
}, 10 * scrapeTimeout);
it.concurrent("filters URLs properly", async () => {
diff --git a/apps/api/src/lib/crawler.ts b/apps/api/src/lib/crawler.ts
index eb281235b..82f0bd3d5 100644
--- a/apps/api/src/lib/crawler.ts
+++ b/apps/api/src/lib/crawler.ts
@@ -88,7 +88,6 @@ class RustCrawler {
export async function filterLinks(
call: FilterLinksCall,
): Promise<FilterLinksResult> {
- console.log(call);
const converter = await RustCrawler.getInstance();
return await converter.filterLinks(call);
}
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 37b576577..9e5adaccf 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -145,37 +145,46 @@ export class WebCrawler {
return { links: sitemapLinks.slice(0, limit), denialReasons };
}
- // NOTE: Temporarily disabling this until we fix the issue with it only returning 1 result sometimes.
- // try {
- // const res = await filterLinks({
- // links: sitemapLinks,
- // limit: isFinite(limit) ? limit : undefined,
- // max_depth: maxDepth,
- // base_url: this.baseUrl,
- // initial_url: this.initialUrl,
- // regex_on_full_url: this.regexOnFullURL,
- // excludes: this.excludes,
- // includes: this.includes,
- // allow_backward_crawling: this.allowBackwardCrawling,
- // ignore_robots_txt: this.ignoreRobotsTxt,
- // robots_txt: this.robotsTxt,
- // });
-
- // const fancyDenialReasons = new Map<string, string>();
- // res.denial_reasons.forEach((value, key) => {
- // fancyDenialReasons.set(key, DenialReason[value]);
- // });
-
- // return {
- // links: res.links,
- // denialReasons: fancyDenialReasons,
- // };
- // } catch (error) {
- // this.logger.error("Error filtering links in Rust, falling back to JS", {
- // error,
- // method: "filterLinks",
- // });
- // }
+ try {
+ const res = await filterLinks({
+ links: sitemapLinks,
+ limit: isFinite(limit) ? limit : undefined,
+ max_depth: maxDepth,
+ base_url: this.baseUrl,
+ initial_url: this.initialUrl,
+ regex_on_full_url: this.regexOnFullURL,
+ excludes: this.excludes,
+ includes: this.includes,
+ allow_backward_crawling: this.allowBackwardCrawling,
+ ignore_robots_txt: this.ignoreRobotsTxt,
+ robots_txt: this.robotsTxt,
+ });
+
+ const fancyDenialReasons = new Map<string, string>();
+ res.denial_reasons.forEach((value, key) => {
+ fancyDenialReasons.set(key, DenialReason[value]);
+ });
+
+ if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {
+ for (const link of res.links) {
+ this.logger.debug(`${link} OK`);
+ }
+
+ for (const [link, reason] of fancyDenialReasons) {
+ this.logger.debug(`${link} ${reason}`);
+ }
+ }
+
+ return {
+ links: res.links,
+ denialReasons: fancyDenialReasons,
+ };
+ } catch (error) {
+ this.logger.error("Error filtering links in Rust, falling back to JS", {
+ error,
+ method: "filterLinks",
+ });
+ }
const filteredLinks = sitemapLinks
.filter((link) => {