Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Optimize map (#2138)
tomba9fbd0ad89a
diff --git a/apps/api/native/Cargo.toml b/apps/api/native/Cargo.tomlindex e08132429..86ef34d4d 100644--- a/apps/api/native/Cargo.toml+++ b/apps/api/native/Cargo.toml@@ -15,6 +15,7 @@ lopdf = "0.38.0"napi = { version = "3.0.0", features = ["serde-json"] }napi-derive = "3.0.0"nodesig = { git = "https://github.com/firecrawl/nodesig" }+psl = "2.1.140"regex = "1.11.2"roxmltree = "0.20.0"serde = { version = "1.0.219", features = ["derive"] }diff --git a/apps/api/native/src/crawler.rs b/apps/api/native/src/crawler.rsindex b1f1b8ce7..399d4e21c 100644--- a/apps/api/native/src/crawler.rs+++ b/apps/api/native/src/crawler.rs@@ -2,10 +2,22 @@ use napi::bindgen_prelude::*;use napi_derive::napi;use regex::Regex;use serde::{Deserialize, Serialize};-use std::{collections::HashMap, usize};+use std::{+ collections::{HashMap, HashSet},+ sync::LazyLock,+};use texting_robots::Robot;use url::Url;+static FILE_EXTENSIONS: &[&str] = &[+ ".png", ".jpg", ".jpeg", ".gif", ".css", ".js", ".ico", ".svg", ".tiff", ".zip", ".exe", ".dmg",+ ".mp4", ".mp3", ".wav", ".pptx", ".xlsx", ".avi", ".flv", ".woff", ".ttf", ".woff2", ".webp",+ ".inc",+];++static FILE_EXT_SET: LazyLock<HashSet<&'static str>> =+ LazyLock::new(|| FILE_EXTENSIONS.iter().copied().collect());+#[derive(Deserialize)]#[napi(object)]pub struct FilterLinksCall {@@ -20,6 +32,8 @@ pub struct FilterLinksCall {pub allow_backward_crawling: bool,pub ignore_robots_txt: bool,pub robots_txt: String,+ pub allow_external_content_links: bool,+ pub allow_subdomains: bool,}#[derive(Serialize)]@@ -29,6 +43,27 @@ pub struct FilterLinksResult {pub denial_reasons: HashMap<String, String>,}+#[derive(Deserialize)]+#[napi(object)]+pub struct FilterUrlCall {+ pub href: String,+ pub url: String,+ pub base_url: String,+ pub excludes: Vec<String>,+ pub ignore_robots_txt: bool,+ pub robots_txt: String,+ pub allow_external_content_links: bool,+ pub allow_subdomains: bool,+}++#[derive(Serialize)]+#[napi(object)]+pub struct FilterUrlResult {+ pub allowed: bool,+ pub url: Option<String>,+ pub denial_reason: Option<String>,+}+#[derive(Serialize, Debug)]#[napi(object)]pub struct SitemapUrl {@@ -75,150 +110,243 @@ pub struct SitemapProcessingResult {pub total_count: u32,}-fn _is_file(url: &Url) -> bool {- let file_extensions = vec![- ".png", ".jpg", ".jpeg", ".gif", ".css", ".js", ".ico", ".svg", ".tiff", ".zip", ".exe",- ".dmg", ".mp4", ".mp3", ".wav", ".pptx", ".xlsx", ".avi", ".flv", ".woff", ".ttf", ".woff2",- ".webp", ".inc",- ];- let url_without_query = url.path().to_lowercase();- file_extensions- .iter()- .any(|ext| url_without_query.ends_with(ext))+const URL_PARSE_ERROR: &str = "URL_PARSE_ERROR";+const DEPTH_LIMIT: &str = "DEPTH_LIMIT";+const EXCLUDE_PATTERN: &str = "EXCLUDE_PATTERN";+const INCLUDE_PATTERN: &str = "INCLUDE_PATTERN";+const BACKWARD_CRAWLING: &str = "BACKWARD_CRAWLING";+const ROBOTS_TXT: &str = "ROBOTS_TXT";+const FILE_TYPE: &str = "FILE_TYPE";+const SOCIAL_MEDIA: &str = "SOCIAL_MEDIA";+const EXTERNAL_LINK: &str = "EXTERNAL_LINK";+const SECTION_LINK: &str = "SECTION_LINK";++#[inline]+fn is_file(path: &str) -> bool {+ if let Some(dot_pos) = path.rfind('.') {+ let extension = &path[dot_pos..];+ FILE_EXT_SET.contains(extension)+ } else {+ false+ }}-fn _get_url_depth(path: &str) -> u32 {+#[inline]+fn get_url_depth(path: &str) -> u32 {path.split('/')- .filter(|x| *x != "" && *x != "index.php" && *x != "index.html")+ .filter(|segment| !segment.is_empty() && *segment != "index.php" && *segment != "index.html").count() as u32}-fn _filter_links(data: FilterLinksCall) -> std::result::Result<FilterLinksResult, String> {- let mut denial_reasons = HashMap::new();+#[inline]+fn is_internal_link(url: &Url, base_url: &Url) -> bool {+ let base_domain = base_url+ .host_str()+ .unwrap_or("")+ .trim_start_matches("www.")+ .trim();+ let link_domain = url+ .host_str()+ .unwrap_or("")+ .trim_start_matches("www.")+ .trim();++ link_domain == base_domain+}++#[inline]+fn no_sections(url_str: &str) -> bool {+ if !url_str.contains('#') {+ return true;+ }++ // Check if the hash fragment looks like a route (contains forward slashes and has substantial content)+ if let Some(hash_part) = url_str.split('#').nth(1) {+ hash_part.len() > 1 && hash_part.contains('/')+ } else {+ false+ }+}++#[inline]+fn is_social_media_or_email(url_str: &str) -> bool {+ const SOCIAL_MEDIA_OR_EMAIL: &[&str] = &[+ "facebook.com",+ "twitter.com",+ "linkedin.com",+ "instagram.com",+ "pinterest.com",+ "mailto:",+ "github.com",+ "calendly.com",+ "discord.gg",+ "discord.com",+ ];++ SOCIAL_MEDIA_OR_EMAIL+ .iter()+ .any(|domain| url_str.contains(domain))+}++#[inline]+fn is_subdomain(url: &Url, base_url: &Url) -> bool {+ match (url.host_str(), base_url.host_str()) {+ (Some(link_host), Some(base_host)) => {+ match (psl::domain_str(link_host), psl::domain_str(base_host)) {+ (Some(link_domain), Some(base_domain)) => link_domain == base_domain,+ _ => false,+ }+ }+ _ => false,+ }+}- let limit = data- .limit- .and_then(|x| if x < 0 { Some(0) } else { Some(x as usize) })- .unwrap_or(usize::MAX);+#[inline]+fn is_external_main_page(url_str: &str) -> bool {+ if let Ok(url) = Url::parse(url_str) {+ let path_segments: Vec<&str> = url+ .path_segments()+ .map(|segments| segments.filter(|s| !s.is_empty()).collect())+ .unwrap_or_default();+ path_segments.is_empty()+ } else {+ false+ }+}+fn _filter_links(data: FilterLinksCall) -> std::result::Result<FilterLinksResult, String> {+ let limit = data.limit.map_or(usize::MAX, |x| x.max(0) as usize);if limit == 0 {return Ok(FilterLinksResult {- links: Vec::with_capacity(0),- denial_reasons,+ links: Vec::new(),+ denial_reasons: HashMap::new(),});}- let base_url = Url::parse(&data.base_url).map_err(|e| format!("Base URL parse error: {}", e))?;+ let base_url = Url::parse(&data.base_url).map_err(|e| format!("Base URL parse error: {e}"))?;let initial_url =- Url::parse(&data.initial_url).map_err(|e| format!("Initial URL parse error: {}", e))?;+ Url::parse(&data.initial_url).map_err(|e| format!("Initial URL parse error: {e}"))?;+ let initial_path = initial_url.path();- let excludes_regex = data+ let excludes_regex: Vec<Regex> = data.excludes.iter()- .map(|exclude| Regex::new(exclude))- .collect::<Vec<std::result::Result<Regex, regex::Error>>>();- let excludes_regex = excludes_regex- .into_iter()- .filter_map(|x| x.ok())- .collect::<Vec<Regex>>();-- let includes_regex = data+ .filter_map(|e| Regex::new(e).ok())+ .collect();+ let includes_regex: Vec<Regex> = data.includes.iter()- .map(|include| Regex::new(include))- .collect::<Vec<std::result::Result<Regex, regex::Error>>>();- let includes_regex = includes_regex- .into_iter()- .filter_map(|x| x.ok())- .collect::<Vec<Regex>>();-- let links = data- .links- .into_iter()- .filter(|link| {- let url = match base_url.join(link) {- Ok(x) => x,- Err(_) => {- denial_reasons.insert(link.clone(), "URL_PARSE_ERROR".to_string());- return false;- }- };+ .filter_map(|i| Regex::new(i).ok())+ .collect();++ let robot = if !data.ignore_robots_txt && !data.robots_txt.is_empty() {+ Robot::new("FireCrawlAgent", data.robots_txt.as_bytes())+ .ok()+ .or_else(|| Robot::new("FirecrawlAgent", data.robots_txt.as_bytes()).ok())+ } else {+ None+ };++ let mut result_links = Vec::new();+ let mut denial_reasons = HashMap::new();++ for link in data.links {+ if result_links.len() >= limit {+ break;+ }++ let url = match base_url.join(&link) {+ Ok(url) => url,+ Err(_) => {+ denial_reasons.insert(link, URL_PARSE_ERROR.to_string());+ continue;+ }+ };++ let path = url.path();+ let url_str = url.as_str();- let path = url.path();- let depth = _get_url_depth(path);- if depth > data.max_depth {- denial_reasons.insert(link.clone(), "DEPTH_LIMIT".to_string());- return false;+ if get_url_depth(path) > data.max_depth {+ denial_reasons.insert(link, DEPTH_LIMIT.to_string());+ continue;+ }++ if is_file(path) {+ denial_reasons.insert(link, FILE_TYPE.to_string());+ continue;+ }++ if is_internal_link(&url, &base_url) {+ // INTERNAL LINKS+ if !no_sections(url_str) {+ denial_reasons.insert(link, SECTION_LINK.to_string());+ continue;}- let exinc_path = if data.regex_on_full_url {- url.as_str()+ if !data.allow_backward_crawling && !path.starts_with(initial_path) {+ denial_reasons.insert(link, BACKWARD_CRAWLING.to_string());+ continue;+ }++ let match_target = if data.regex_on_full_url {+ url_str} else {- url.path()+ path};- if !excludes_regex.is_empty()- && excludes_regex- .iter()- .any(|regex| regex.is_match(exinc_path))- {- denial_reasons.insert(link.clone(), "EXCLUDE_PATTERN".to_string());- return false;+ if !excludes_regex.is_empty() && excludes_regex.iter().any(|r| r.is_match(match_target)) {+ denial_reasons.insert(link, EXCLUDE_PATTERN.to_string());+ continue;}- if !includes_regex.is_empty()- && !includes_regex- .iter()- .any(|regex| regex.is_match(exinc_path))- {- denial_reasons.insert(link.clone(), "INCLUDE_PATTERN".to_string());- return false;+ if !includes_regex.is_empty() && !includes_regex.iter().any(|r| r.is_match(match_target)) {+ denial_reasons.insert(link, INCLUDE_PATTERN.to_string());+ continue;}- if !data.allow_backward_crawling {- if !url.path().starts_with(initial_url.path()) {- denial_reasons.insert(link.clone(), "BACKWARD_CRAWLING".to_string());- return false;+ if let Some(ref robot) = robot {+ if !robot.allowed(url_str) {+ denial_reasons.insert(link, ROBOTS_TXT.to_string());+ continue;}}- if !data.ignore_robots_txt {- match Robot::new("FireCrawlAgent", data.robots_txt.as_bytes()) {- Ok(robot) => {- let allowed = robot.allowed(url.as_str());- if !allowed {- match Robot::new("FirecrawlAgent", data.robots_txt.as_bytes()) {- Ok(robot_alt) => {- let allowed_alt = robot_alt.allowed(url.as_str());- if !allowed_alt {- denial_reasons.insert(link.clone(), "ROBOTS_TXT".to_string());- return false;- }- }- Err(_) => {- denial_reasons.insert(link.clone(), "ROBOTS_TXT".to_string());- return false;- }- }- }- }- Err(_) => {}- }+ result_links.push(link);+ } else {+ // EXTERNAL LINKS+ if is_social_media_or_email(url_str) {+ denial_reasons.insert(link, SOCIAL_MEDIA.to_string());+ continue;}- if _is_file(&url) {- denial_reasons.insert(link.clone(), "FILE_TYPE".to_string());- return false;+ if !excludes_regex.is_empty() && excludes_regex.iter().any(|r| r.is_match(url_str)) {+ denial_reasons.insert(link, EXCLUDE_PATTERN.to_string());+ continue;}- true- })- .take(limit)- .collect::<Vec<_>>();+ if is_internal_link(&initial_url, &base_url)+ && data.allow_external_content_links+ && !is_external_main_page(url_str)+ {+ result_links.push(link);+ continue;+ }++ if data.allow_subdomains+ && !is_social_media_or_email(url_str)+ && is_subdomain(&url, &base_url)+ {+ result_links.push(link);+ continue;+ }++ denial_reasons.insert(link, EXTERNAL_LINK.to_string());+ }+ }Ok(FilterLinksResult {- links,+ links: result_links,denial_reasons,})}@@ -227,7 +355,169 @@ fn _filter_links(data: FilterLinksCall) -> std::result::Result<FilterLinksResult#[napi]pub fn filter_links(data: FilterLinksCall) -> Result<FilterLinksResult> {_filter_links(data)- .map_err(|e| Error::new(Status::GenericFailure, format!("Filter links error: {}", e)))+ .map_err(|e| Error::new(Status::GenericFailure, format!("Filter links error: {e}")))+}++fn _filter_url(data: FilterUrlCall) -> std::result::Result<FilterUrlResult, String> {+ let mut full_url = data.href.clone();++ // Handle relative URLs+ if !data.href.starts_with("http") {+ match Url::parse(&data.url) {+ Ok(base) => match base.join(&data.href) {+ Ok(resolved) => full_url = resolved.to_string(),+ Err(_) => {+ return Ok(FilterUrlResult {+ allowed: false,+ url: None,+ denial_reason: Some(URL_PARSE_ERROR.to_string()),+ });+ }+ },+ Err(_) => {+ return Ok(FilterUrlResult {+ allowed: false,+ url: None,+ denial_reason: Some(URL_PARSE_ERROR.to_string()),+ });+ }+ }+ }++ let url = match Url::parse(&full_url) {+ Ok(url) => url,+ Err(_) => {+ return Ok(FilterUrlResult {+ allowed: false,+ url: None,+ denial_reason: Some(URL_PARSE_ERROR.to_string()),+ });+ }+ };++ let base_url = match Url::parse(&data.base_url) {+ Ok(url) => url,+ Err(_) => {+ return Ok(FilterUrlResult {+ allowed: false,+ url: None,+ denial_reason: Some(URL_PARSE_ERROR.to_string()),+ });+ }+ };++ let path = url.path();+ let url_str = url.as_str();++ let excludes_regex: Vec<Regex> = data+ .excludes+ .iter()+ .filter_map(|e| Regex::new(e).ok())+ .collect();++ let robot = if !data.ignore_robots_txt && !data.robots_txt.is_empty() {+ Robot::new("FireCrawlAgent", data.robots_txt.as_bytes())+ .ok()+ .or_else(|| Robot::new("FirecrawlAgent", data.robots_txt.as_bytes()).ok())+ } else {+ None+ };++ if is_internal_link(&url, &base_url) {+ // INTERNAL LINKS+ if !no_sections(url_str) {+ return Ok(FilterUrlResult {+ allowed: false,+ url: None,+ denial_reason: Some(SECTION_LINK.to_string()),+ });+ }++ if !excludes_regex.is_empty() && excludes_regex.iter().any(|r| r.is_match(path)) {+ return Ok(FilterUrlResult {+ allowed: false,+ url: None,+ denial_reason: Some(EXCLUDE_PATTERN.to_string()),+ });+ }++ if let Some(ref robot) = robot {+ if !robot.allowed(url_str) {+ return Ok(FilterUrlResult {+ allowed: false,+ url: None,+ denial_reason: Some(ROBOTS_TXT.to_string()),+ });+ }+ }++ Ok(FilterUrlResult {+ allowed: true,+ url: Some(full_url),+ denial_reason: None,+ })+ } else {+ // EXTERNAL LINKS+ if is_social_media_or_email(url_str) {+ return Ok(FilterUrlResult {+ allowed: false,+ url: None,+ denial_reason: Some(SOCIAL_MEDIA.to_string()),+ });+ }++ if !excludes_regex.is_empty() && excludes_regex.iter().any(|r| r.is_match(url_str)) {+ return Ok(FilterUrlResult {+ allowed: false,+ url: None,+ denial_reason: Some(EXCLUDE_PATTERN.to_string()),+ });+ }++ let context_url = match Url::parse(&data.url) {+ Ok(url) => url,+ Err(_) => {+ return Ok(FilterUrlResult {+ allowed: false,+ url: None,+ denial_reason: Some(URL_PARSE_ERROR.to_string()),+ });+ }+ };++ if is_internal_link(&context_url, &base_url)+ && data.allow_external_content_links+ && !is_external_main_page(url_str)+ {+ return Ok(FilterUrlResult {+ allowed: true,+ url: Some(full_url),+ denial_reason: None,+ });+ }++ if data.allow_subdomains && !is_social_media_or_email(url_str) && is_subdomain(&url, &base_url)+ {+ return Ok(FilterUrlResult {+ allowed: true,+ url: Some(full_url),+ denial_reason: None,+ });+ }++ Ok(FilterUrlResult {+ allowed: false,+ url: None,+ denial_reason: Some(EXTERNAL_LINK.to_string()),+ })+ }+}++/// Filter a single URL based on crawling rules and constraints.+#[napi]+pub fn filter_url(data: FilterUrlCall) -> Result<FilterUrlResult> {+ _filter_url(data)+ .map_err(|e| Error::new(Status::GenericFailure, format!("Filter URL error: {e}")))}fn _parse_sitemap_xml(xml_content: &str) -> std::result::Result<ParsedSitemap, String> {@@ -238,28 +528,24 @@ fn _parse_sitemap_xml(xml_content: &str) -> std::result::Result<ParsedSitemap, S..Default::default()},)- .map_err(|e| format!("XML parsing error: {}", e))?;+ .map_err(|e| format!("XML parsing error: {e}"))?;let root = doc.root_element();match root.tag_name().name() {"sitemapindex" => {- let mut sitemaps = Vec::new();-- for sitemap_node in root+ let sitemaps = root.children().filter(|n| n.is_element() && n.tag_name().name() == "sitemap")- {- if let Some(loc_node) = sitemap_node- .children()- .find(|n| n.is_element() && n.tag_name().name() == "loc")- {- if let Some(loc_text) = loc_node.text() {- sitemaps.push(SitemapEntry {+ .filter_map(|sitemap_node| {+ sitemap_node+ .children()+ .find(|n| n.is_element() && n.tag_name().name() == "loc")+ .and_then(|loc_node| loc_node.text())+ .map(|loc_text| SitemapEntry {loc: vec![loc_text.to_string()],- });- }- }- }+ })+ })+ .collect();Ok(ParsedSitemap {urlset: None,@@ -267,23 +553,19 @@ fn _parse_sitemap_xml(xml_content: &str) -> std::result::Result<ParsedSitemap, S})}"urlset" => {- let mut urls = Vec::new();-- for url_node in root+ let urls = root.children().filter(|n| n.is_element() && n.tag_name().name() == "url")- {- if let Some(loc_node) = url_node- .children()- .find(|n| n.is_element() && n.tag_name().name() == "loc")- {- if let Some(loc_text) = loc_node.text() {- urls.push(SitemapUrl {+ .filter_map(|url_node| {+ url_node+ .children()+ .find(|n| n.is_element() && n.tag_name().name() == "loc")+ .and_then(|loc_node| loc_node.text())+ .map(|loc_text| SitemapUrl {loc: vec![loc_text.to_string()],- });- }- }- }+ })+ })+ .collect();Ok(ParsedSitemap {urlset: Some(SitemapUrlset { url: urls }),@@ -300,7 +582,7 @@ pub fn parse_sitemap_xml(xml_content: String) -> Result<ParsedSitemap> {_parse_sitemap_xml(&xml_content).map_err(|e| {Error::new(Status::GenericFailure,- format!("Parse sitemap XML error: {}", e),+ format!("Parse sitemap XML error: {e}"),)})}@@ -324,12 +606,13 @@ fn _process_sitemap(xml_content: &str) -> std::result::Result<SitemapProcessingR.collect();if !sitemap_urls.is_empty() {+ let count = sitemap_urls.len() as u32;instructions.push(SitemapInstruction {action: "recurse".to_string(),- urls: sitemap_urls.clone(),- count: sitemap_urls.len() as u32,+ urls: sitemap_urls,+ count,});- total_count += sitemap_urls.len() as u32;+ total_count += count;}} else if let Some(urlset) = parsed.urlset {let mut xml_sitemaps = Vec::new();@@ -338,10 +621,12 @@ fn _process_sitemap(xml_content: &str) -> std::result::Result<SitemapProcessingRfor url_entry in urlset.url {if !url_entry.loc.is_empty() {let url = url_entry.loc[0].trim();- if url.to_lowercase().ends_with(".xml") || url.to_lowercase().ends_with(".xml.gz") {+ let url_lower = url.to_lowercase();+ if url_lower.ends_with(".xml") || url_lower.ends_with(".xml.gz") {xml_sitemaps.push(url.to_string());} else if let Ok(parsed_url) = Url::parse(url) {- if !_is_file(&parsed_url) {+ let path_lower = parsed_url.path().to_lowercase();+ if !is_file(&path_lower) {valid_urls.push(url.to_string());}}@@ -349,21 +634,23 @@ fn _process_sitemap(xml_content: &str) -> std::result::Result<SitemapProcessingR}if !xml_sitemaps.is_empty() {+ let count = xml_sitemaps.len() as u32;instructions.push(SitemapInstruction {action: "recurse".to_string(),- urls: xml_sitemaps.clone(),- count: xml_sitemaps.len() as u32,+ urls: xml_sitemaps,+ count,});- total_count += xml_sitemaps.len() as u32;+ total_count += count;}if !valid_urls.is_empty() {+ let count = valid_urls.len() as u32;instructions.push(SitemapInstruction {action: "process".to_string(),- urls: valid_urls.clone(),- count: valid_urls.len() as u32,+ urls: valid_urls,+ count,});- total_count += valid_urls.len() as u32;+ total_count += count;}}@@ -379,7 +666,7 @@ pub fn process_sitemap(xml_content: String) -> Result<SitemapProcessingResult> {_process_sitemap(&xml_content).map_err(|e| {Error::new(Status::GenericFailure,- format!("Process sitemap error: {}", e),+ format!("Process sitemap error: {e}"),)})}@@ -445,10 +732,7 @@ mod tests {let result = _parse_sitemap_xml(xml_content);assert!(result.is_err());- assert!(result- .unwrap_err()- .to_string()- .contains("Invalid sitemap format"));+ assert!(result.unwrap_err().contains("Invalid sitemap format"));}#[test]@@ -545,6 +829,8 @@ mod tests {initial_url: "https://example.com".to_string(),regex_on_full_url: false,allow_backward_crawling: true,+ allow_external_content_links: false,+ allow_subdomains: false,};let result = _filter_links(data).unwrap();@@ -577,6 +863,8 @@ mod tests {initial_url: "https://example.com".to_string(),regex_on_full_url: false,allow_backward_crawling: true,+ allow_external_content_links: false,+ allow_subdomains: false,};let result = _filter_links(data);@@ -604,6 +892,8 @@ mod tests {initial_url: "https://example.com".to_string(),regex_on_full_url: false,allow_backward_crawling: true,+ allow_external_content_links: false,+ allow_subdomains: false,};let result = _filter_links(data);@@ -629,6 +919,8 @@ mod tests {initial_url: "https://example.com".to_string(),regex_on_full_url: false,allow_backward_crawling: true,+ allow_external_content_links: false,+ allow_subdomains: false,};let result = _filter_links(data);@@ -637,4 +929,13 @@ mod tests {assert_eq!(result.links.len(), 1);assert_eq!(result.links[0], "https://example.com/test");}++ #[test]+ fn test_is_file() {+ assert!(is_file("test.png"));+ assert!(is_file("script.js"));+ assert!(is_file("style.css"));+ assert!(!is_file("page"));+ assert!(!is_file("directory/"));+ }}diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.tsindex aaf966b1a..40d85b067 100644--- a/apps/api/src/controllers/v2/map.ts+++ b/apps/api/src/controllers/v2/map.ts@@ -45,16 +45,19 @@ interface MapResult {}function dedupeMapDocumentArray(documents: MapDocument[]): MapDocument[] {- let urlSet = new Set<string>(documents.map(x => x.url));+ const urlMap = new Map<string, MapDocument>();- let newDocuments: MapDocument[] = [- ...Array.from(urlSet).map(x => {- let localDocs = documents.filter(y => y.url === x);- return localDocs.find(x => x.title !== undefined) || localDocs[0];- }),- ];+ for (const doc of documents) {+ const existing = urlMap.get(doc.url);++ if (!existing) {+ urlMap.set(doc.url, doc);+ } else if (doc.title !== undefined && existing.title === undefined) {+ urlMap.set(doc.url, doc);+ }+ }- return newDocuments;+ return Array.from(urlMap.values());}async function queryIndex(@@ -116,6 +119,8 @@ async function getMapResults({useIndex?: boolean;location?: ScrapeOptions["location"];}): Promise<MapResult> {+ const functionStartTime = Date.now();+const id = uuidv4();let mapResults: MapDocument[] = [];const zeroDataRetention = flags?.forceZDR ?? false;@@ -141,7 +146,9 @@ async function getMapResults({try {sc.robots = await crawler.getRobotsTxt(false, abort);crawler.importRobotsTxt(sc.robots);- } catch (_) {}+ } catch (_) {+ // Robots.txt fetch failed, continue without it+ }// If sitemapOnly is true, only get links from sitemapif (crawlerOptions.sitemap === "only") {@@ -159,6 +166,7 @@ async function getMapResults({abort,crawlerOptions.useMock,);+if (sitemap > 0) {mapResults = mapResults.slice(1)@@ -176,7 +184,6 @@ async function getMapResults({}} else {let urlWithoutWww = url.replace("www.", "");-let mapUrl =search && allowExternalLinks? `${search} ${urlWithoutWww}`@@ -198,7 +205,7 @@ async function getMapResults({pagePromises = JSON.parse(cachedResult);} else {const fetchPage = async (page: number) => {- return fireEngineMap(+ return await fireEngineMap(mapUrl,{numResults: resultsPerPage,@@ -213,7 +220,6 @@ async function getMapResults({);}- // Parallelize sitemap index query with search resultsconst [indexResults, searchResults] = await Promise.all([queryIndex(url, limit, useIndex, includeSubdomains),Promise.all(pagePromises),@@ -232,8 +238,6 @@ async function getMapResults({mapResults.push(...indexResults);}- // If sitemap is not ignored, fetch sitemap- // This will attempt to find it in the index at first, or fetch a fresh one if it's older than 2 daysif (crawlerOptions.sitemap === "include") {try {await crawler.tryGetSitemap(@@ -285,66 +289,64 @@ async function getMapResults({const searchQuery = search.toLowerCase();mapResults = performCosineSimilarityV2(mapResults, searchQuery);}+ }- mapResults = mapResults- .map(x => {- try {- return {- ...x,- url: checkAndUpdateURLForMap(- x.url,- crawlerOptions.ignoreQueryParameters ?? true,- ).url.trim(),- };- } catch (_) {- return null;- }- })- .filter(x => x !== null) as MapDocument[];-- // allows for subdomains to be included- mapResults = mapResults.filter(x => isSameDomain(x.url, url));-- // if includeSubdomains is false, filter out subdomains- if (!includeSubdomains) {- mapResults = mapResults.filter(x => isSameSubdomain(x.url, url));- }-- // Filter by path if enabled- if (filterByPath && !allowExternalLinks) {+ mapResults = mapResults+ .map(x => {try {- const urlObj = new URL(url);- const urlPath = urlObj.pathname;- // Only apply path filtering if the URL has a significant path (not just '/' or empty)- // This means we only filter by path if the user has not selected a root domain- if (urlPath && urlPath !== "/" && urlPath.length > 1) {- mapResults = mapResults.filter(x => {- try {- const linkObj = new URL(x.url);- return linkObj.pathname.startsWith(urlPath);- } catch (e) {- return false;- }- });- }- } catch (e) {- // If URL parsing fails, continue without path filtering- logger.warn(`Failed to parse URL for path filtering: ${url}`, {- error: e,+ return {+ ...x,+ url: checkAndUpdateURLForMap(+ x.url,+ crawlerOptions.ignoreQueryParameters ?? true,+ ).url.trim(),+ };+ } catch (_) {+ return null;+ }+ })+ .filter(x => x !== null) as MapDocument[];++ mapResults = mapResults.filter(x => isSameDomain(x.url, url));++ if (!includeSubdomains) {+ mapResults = mapResults.filter(x => isSameSubdomain(x.url, url));+ }++ if (filterByPath && !allowExternalLinks) {+ try {+ const urlObj = new URL(url);+ const urlPath = urlObj.pathname;+ // Only apply path filtering if the URL has a significant path (not just '/' or empty)+ // This means we only filter by path if the user has not selected a root domain+ if (urlPath && urlPath !== "/" && urlPath.length > 1) {+ mapResults = mapResults.filter(x => {+ try {+ const linkObj = new URL(x.url);+ return linkObj.pathname.startsWith(urlPath);+ } catch (e) {+ return false;+ }});}+ } catch (e) {+ // If URL parsing fails, continue without path filtering+ logger.warn(`Failed to parse URL for path filtering: ${url}`, {+ error: e,+ });}-- mapResults = dedupeMapDocumentArray(mapResults);}+ mapResults = dedupeMapDocumentArray(mapResults);mapResults = mapResults.slice(0, limit);+ const totalTimeMs = Date.now() - functionStartTime;+return {success: true,mapResults,job_id: id,- time_taken: (new Date().getTime() - Date.now()) / 1000,+ time_taken: totalTimeMs,};}diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex c58a148c8..b86cbf355 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -15,7 +15,7 @@ import {} from "../../lib/robots-txt";import { ScrapeJobTimeoutError } from "../../lib/error";import { ScrapeOptions } from "../../controllers/v2/types";-import { filterLinks } from "@mendable/firecrawl-rs";+import { filterLinks, filterUrl } from "@mendable/firecrawl-rs";export const SITEMAP_LIMIT = 100;@@ -175,6 +175,8 @@ export class WebCrawler {allowBackwardCrawling: this.allowBackwardCrawling,ignoreRobotsTxt: this.ignoreRobotsTxt,robotsTxt: this.robotsTxt,+ allowExternalContentLinks: this.allowExternalContentLinks,+ allowSubdomains: this.allowSubdomains,});const fancyDenialReasons = new Map<string, string>();@@ -427,32 +429,36 @@ export class WebCrawler {return await urlsHandler(urls);} else {let filteredLinksResult = await this.filterLinks(- [...new Set(urls)].filter(- x => this.filterURL(x, this.initialUrl).allowed,- ),+ [...new Set(urls)],leftOfLimit,this.maxCrawledDepth,fromMap,);let filteredLinks = filteredLinksResult.links;leftOfLimit -= filteredLinks.length;- let uniqueURLs: string[] = [];- for (const url of filteredLinks) {- if (- await redisEvictConnection.sadd(- "sitemap:" + this.jobId + ":links",- normalizeUrl(url),- )- ) {- uniqueURLs.push(url);- }- }+ const pipeline = redisEvictConnection.pipeline();++ const normalizedUrls = filteredLinks.map(url => normalizeUrl(url));+ normalizedUrls.forEach(normalizedUrl => {+ pipeline.sadd("sitemap:" + this.jobId + ":links", normalizedUrl);+ });++ const results = await pipeline.exec();++ const uniqueURLs = filteredLinks.filter(+ (_, index) =>+ results &&+ results[index] &&+ !results[index][0] &&+ results[index][1] === 1,+ );await redisEvictConnection.expire("sitemap:" + this.jobId + ":links",3600,"NX",);+if (uniqueURLs.length > 0) {return await urlsHandler(uniqueURLs);}@@ -528,83 +534,24 @@ export class WebCrawler {}}- public filterURL(href: string, url: string): FilterResult {- let fullUrl = href;- if (!href.startsWith("http")) {- try {- fullUrl = new URL(href, url).toString();- } catch (_) {- return { allowed: false, denialReason: DenialReason.URL_PARSE_ERROR };- }- }- let urlObj;- try {- urlObj = new URL(fullUrl);- } catch (_) {- return { allowed: false, denialReason: DenialReason.URL_PARSE_ERROR };- }- const path = urlObj.pathname;-- if (this.isInternalLink(fullUrl)) {- // INTERNAL LINKS- if (!this.noSections(fullUrl)) {- return { allowed: false, denialReason: DenialReason.SECTION_LINK };- }-- if (this.matchesExcludes(path)) {- return { allowed: false, denialReason: DenialReason.EXCLUDE_PATTERN };- }-- if (!this.isRobotsAllowed(fullUrl, this.ignoreRobotsTxt)) {- (async () => {- await redisEvictConnection.sadd(- "crawl:" + this.jobId + ":robots_blocked",- fullUrl,- );- await redisEvictConnection.expire(- "crawl:" + this.jobId + ":robots_blocked",- 24 * 60 * 60,- );- })();- return { allowed: false, denialReason: DenialReason.ROBOTS_TXT };- }-- return { allowed: true, url: fullUrl };- } else {- // EXTERNAL LINKS- if (this.isSocialMediaOrEmail(fullUrl)) {- return { allowed: false, denialReason: DenialReason.SOCIAL_MEDIA };- }-- if (this.matchesExcludes(fullUrl, true)) {- return { allowed: false, denialReason: DenialReason.EXCLUDE_PATTERN };- }-- if (- this.isInternalLink(url) &&- this.allowExternalContentLinks &&- !this.isExternalMainPage(fullUrl)- ) {- return { allowed: true, url: fullUrl };- }-- if (- this.allowSubdomains &&- !this.isSocialMediaOrEmail(fullUrl) &&- this.isSubdomain(fullUrl)- ) {- return { allowed: true, url: fullUrl };- }-- return { allowed: false, denialReason: DenialReason.EXTERNAL_LINK };- }+ public async filterURL(href: string, url: string): Promise<FilterResult> {+ return await filterUrl({+ href: href,+ url: url,+ baseUrl: this.baseUrl,+ excludes: this.excludes,+ ignoreRobotsTxt: this.ignoreRobotsTxt,+ robotsTxt: this.robotsTxt,+ allowExternalContentLinks: this.allowExternalContentLinks,+ allowSubdomains: this.allowSubdomains,+ });}private async extractLinksFromHTMLRust(html: string, url: string) {const links = await extractLinks(html);const filteredLinks: string[] = [];for (const link of links) {- const filterResult = this.filterURL(link, url);+ const filterResult = await this.filterURL(link, url);if (filterResult.allowed && filterResult.url) {filteredLinks.push(filterResult.url);}@@ -612,32 +559,37 @@ export class WebCrawler {return filteredLinks;}- private extractLinksFromHTMLCheerio(html: string, url: string) {+ private async extractLinksFromHTMLCheerio(html: string, url: string) {let links: string[] = [];const $ = load(html);- $("a").each((_, element) => {+ for (let i = 0; i < $("a").length; i++) {+ const element = $("a")[i];let href = $(element).attr("href");if (href) {if (href.match(/^https?:\/[^\/]/)) {href = href.replace(/^https?:\//, "$&/");}- const filterResult = this.filterURL(href, url);+ const filterResult = await this.filterURL(href, url);if (filterResult.allowed && filterResult.url) {links.push(filterResult.url);}}- });+ }// Extract links from iframes with inline src- $("iframe").each((_, element) => {+ for (let i = 0; i < $("iframe").length; i++) {+ const element = $("iframe")[i];const src = $(element).attr("src");if (src && src.startsWith("data:text/html")) {const iframeHtml = decodeURIComponent(src.split(",")[1]);- const iframeLinks = this.extractLinksFromHTMLCheerio(iframeHtml, url);+ const iframeLinks = await this.extractLinksFromHTMLCheerio(+ iframeHtml,+ url,+ );links = links.concat(iframeLinks);}- });+ }return links;}@@ -668,7 +620,7 @@ export class WebCrawler {);}- return this.extractLinksFromHTMLCheerio(html, url);+ return await this.extractLinksFromHTMLCheerio(html, url);}private isRobotsAllowed(@@ -678,94 +630,6 @@ export class WebCrawler {return ignoreRobotsTxt ? true : isUrlAllowedByRobots(url, this.robots);}- private matchesExcludes(url: string, onlyDomains: boolean = false): boolean {- return this.excludes.some(pattern => {- if (onlyDomains) return this.matchesExcludesExternalDomains(url);-- return this.excludes.some(pattern => new RegExp(pattern).test(url));- });- }-- // supported formats: "example.com/blog", "https://example.com", "blog.example.com", "example.com"- private matchesExcludesExternalDomains(url: string) {- try {- const urlObj = new URL(url);- const hostname = urlObj.hostname;- const pathname = urlObj.pathname;-- for (let domain of this.excludes) {- let domainObj = new URL("http://" + domain.replace(/^https?:\/\//, ""));- let domainHostname = domainObj.hostname;- let domainPathname = domainObj.pathname;-- if (- hostname === domainHostname ||- hostname.endsWith(`.${domainHostname}`)- ) {- if (pathname.startsWith(domainPathname)) {- return true;- }- }- }- return false;- } catch (e) {- return false;- }- }-- private isExternalMainPage(url: string): boolean {- return !Boolean(- url- .split("/")- .slice(3)- .filter(subArray => subArray.length > 0).length,- );- }-- private noSections(link: string): boolean {- // Allow URLs with hash fragments that represent actual routes/pages (like SPAs)- // but block simple anchor links within the same page- if (!link.includes("#")) {- return true;- }-- // Check if the hash fragment looks like a route (contains forward slashes and has substantial content)- const hashPart = link.split("#")[1];- if (hashPart && hashPart.length > 1 && hashPart.includes("/")) {- return true;- }-- return false;- }-- private isInternalLink(link: string): boolean {- const urlObj = new URL(link, this.baseUrl);- const baseDomain = new URL(this.baseUrl).hostname- .replace(/^www\./, "")- .trim();- const linkDomain = urlObj.hostname.replace(/^www\./, "").trim();-- return linkDomain === baseDomain;- }-- private isSubdomain(link: string): boolean {- try {- const linkUrl = new URL(link, this.baseUrl);- const baseUrl = new URL(this.baseUrl);-- const linkParsed = psl.parse(linkUrl.hostname);- const baseParsed = psl.parse(baseUrl.hostname);-- if (!linkParsed?.domain || !baseParsed?.domain) {- return false;- }-- return linkParsed.domain === baseParsed.domain;- } catch (error) {- return false;- }- }-public isFile(url: string): boolean {const fileExtensions = [".png",@@ -809,22 +673,6 @@ export class WebCrawler {}}- private isSocialMediaOrEmail(url: string): boolean {- const socialMediaOrEmail = [- "facebook.com",- "twitter.com",- "linkedin.com",- "instagram.com",- "pinterest.com",- "mailto:",- "github.com",- "calendly.com",- "discord.gg",- "discord.com",- ];- return socialMediaOrEmail.some(ext => url.includes(ext));- }-private async tryFetchSitemapLinks(url: string,urlsHandler: (urls: string[]) => unknown,diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 72dff9795..02efe206c 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -72,6 +72,7 @@ import {ScrapeJobSingleUrls,} from "../../types";import { scrapeSitemap } from "../../scraper/crawler/sitemap";+import { filterUrl } from "@mendable/firecrawl-rs";const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));@@ -254,7 +255,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {sc,(await getACUCTeam(job.data.team_id))?.flags ?? null,);- const filterResult = crawler.filterURL(+ const filterResult = await crawler.filterURL(doc.metadata.url,doc.metadata.sourceURL,);@@ -696,7 +697,7 @@ async function kickoffGetIndexLinks();const validIndexLinksResult = await crawler.filterLinks(- index.filter(x => crawler.filterURL(x, trimmedURL.href).allowed),+ index,sc.crawlerOptions.limit ?? 10000,sc.crawlerOptions.maxDepth ?? 10,false,@@ -955,9 +956,7 @@ async function processKickoffSitemapJob(job: NuQJob<ScrapeJobKickoffSitemap>) {const passingURLs = (await crawler.filterLinks(- [...new Set(results.urls.map(x => x.href))].filter(- url => crawler.filterURL(url, sc.originUrl!).allowed,- ),+ results.urls.map(x => x.href),Infinity,sc.crawlerOptions.maxDepth ?? 10,false,