Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Restore Rust link filtering logic (#1822)
Gergő Móricz17241e098349
diff --git a/apps/api/sharedLibs/crawler/src/lib.rs b/apps/api/sharedLibs/crawler/src/lib.rsindex 339b87ad4..22f559108 100644--- a/apps/api/sharedLibs/crawler/src/lib.rs+++ b/apps/api/sharedLibs/crawler/src/lib.rs@@ -1,4 +1,4 @@-use std::collections::HashMap;+use std::{collections::HashMap, usize};use serde::{Deserialize, Serialize};use std::ffi::{CStr, CString};use url::Url;@@ -60,16 +60,16 @@ fn _get_url_depth(path: &str) -> u32 {fn _filter_links(data: FilterLinksCall) -> Result<FilterLinksResult, Box<dyn std::error::Error>> {let mut denial_reasons = HashMap::new();- let limit = data.limit.and_then(|x| if x < 0 { Some(0) } else { Some(x as usize) });+ let limit = data.limit+ .and_then(|x| if x < 0 { Some(0) } else { Some(x as usize) })+ .unwrap_or(usize::MAX);- let limit = if let Some(limit) = limit {- limit- } else {+ if limit == 0 {return Ok(FilterLinksResult {links: Vec::with_capacity(0),denial_reasons,});- };+ }let base_url = Url::parse(&data.base_url)?;let initial_url = Url::parse(&data.initial_url)?;diff --git a/apps/api/src/__tests__/snips/crawl.test.ts b/apps/api/src/__tests__/snips/crawl.test.tsindex 907018350..93bd96d04 100644--- a/apps/api/src/__tests__/snips/crawl.test.ts+++ b/apps/api/src/__tests__/snips/crawl.test.ts@@ -13,10 +13,22 @@ beforeAll(async () => {describe("Crawl tests", () => {it.concurrent("works", async () => {- await crawl({+ const results = await crawl({+ url: "https://firecrawl.dev",+ limit: 10,+ }, identity);++ expect(results.completed).toBe(10);+ }, 10 * scrapeTimeout);++ it.concurrent("works with ignoreSitemap: true", async () => {+ const results = await crawl({url: "https://firecrawl.dev",limit: 10,+ ignoreSitemap: true,}, identity);++ expect(results.completed).toBe(10);}, 10 * scrapeTimeout);it.concurrent("filters URLs properly", async () => {diff --git a/apps/api/src/lib/crawler.ts b/apps/api/src/lib/crawler.tsindex eb281235b..82f0bd3d5 100644--- a/apps/api/src/lib/crawler.ts+++ b/apps/api/src/lib/crawler.ts@@ -88,7 +88,6 @@ class RustCrawler {export async function filterLinks(call: FilterLinksCall,): Promise<FilterLinksResult> {- console.log(call);const converter = await RustCrawler.getInstance();return await converter.filterLinks(call);}diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex 37b576577..9e5adaccf 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -145,37 +145,46 @@ export class WebCrawler {return { links: sitemapLinks.slice(0, limit), denialReasons };}- // NOTE: Temporarily disabling this until we fix the issue with it only returning 1 result sometimes.- // try {- // const res = await filterLinks({- // links: sitemapLinks,- // limit: isFinite(limit) ? limit : undefined,- // max_depth: maxDepth,- // base_url: this.baseUrl,- // initial_url: this.initialUrl,- // regex_on_full_url: this.regexOnFullURL,- // excludes: this.excludes,- // includes: this.includes,- // allow_backward_crawling: this.allowBackwardCrawling,- // ignore_robots_txt: this.ignoreRobotsTxt,- // robots_txt: this.robotsTxt,- // });-- // const fancyDenialReasons = new Map<string, string>();- // res.denial_reasons.forEach((value, key) => {- // fancyDenialReasons.set(key, DenialReason[value]);- // });-- // return {- // links: res.links,- // denialReasons: fancyDenialReasons,- // };- // } catch (error) {- // this.logger.error("Error filtering links in Rust, falling back to JS", {- // error,- // method: "filterLinks",- // });- // }+ try {+ const res = await filterLinks({+ links: sitemapLinks,+ limit: isFinite(limit) ? limit : undefined,+ max_depth: maxDepth,+ base_url: this.baseUrl,+ initial_url: this.initialUrl,+ regex_on_full_url: this.regexOnFullURL,+ excludes: this.excludes,+ includes: this.includes,+ allow_backward_crawling: this.allowBackwardCrawling,+ ignore_robots_txt: this.ignoreRobotsTxt,+ robots_txt: this.robotsTxt,+ });++ const fancyDenialReasons = new Map<string, string>();+ res.denial_reasons.forEach((value, key) => {+ fancyDenialReasons.set(key, DenialReason[value]);+ });++ if (process.env.FIRECRAWL_DEBUG_FILTER_LINKS) {+ for (const link of res.links) {+ this.logger.debug(`${link} OK`);+ }++ for (const [link, reason] of fancyDenialReasons) {+ this.logger.debug(`${link} ${reason}`);+ }+ }++ return {+ links: res.links,+ denialReasons: fancyDenialReasons,+ };+ } catch (error) {+ this.logger.error("Error filtering links in Rust, falling back to JS", {+ error,+ method: "filterLinks",+ });+ }const filteredLinks = sitemapLinks.filter((link) => {