Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Optimize map (#2138)

tomba9fbd0ad89a
diff --git a/apps/api/native/Cargo.toml b/apps/api/native/Cargo.toml
index e08132429..86ef34d4d 100644
--- a/apps/api/native/Cargo.toml
+++ b/apps/api/native/Cargo.toml
@@ -15,6 +15,7 @@ lopdf = "0.38.0"
napi = { version = "3.0.0", features = ["serde-json"] }
napi-derive = "3.0.0"
nodesig = { git = "https://github.com/firecrawl/nodesig" }
+psl = "2.1.140"
regex = "1.11.2"
roxmltree = "0.20.0"
serde = { version = "1.0.219", features = ["derive"] }
diff --git a/apps/api/native/src/crawler.rs b/apps/api/native/src/crawler.rs
index b1f1b8ce7..399d4e21c 100644
--- a/apps/api/native/src/crawler.rs
+++ b/apps/api/native/src/crawler.rs
@@ -2,10 +2,22 @@ use napi::bindgen_prelude::*;
use napi_derive::napi;
use regex::Regex;
use serde::{Deserialize, Serialize};
-use std::{collections::HashMap, usize};
+use std::{
+ collections::{HashMap, HashSet},
+ sync::LazyLock,
+};
use texting_robots::Robot;
use url::Url;
+static FILE_EXTENSIONS: &[&str] = &[
+ ".png", ".jpg", ".jpeg", ".gif", ".css", ".js", ".ico", ".svg", ".tiff", ".zip", ".exe", ".dmg",
+ ".mp4", ".mp3", ".wav", ".pptx", ".xlsx", ".avi", ".flv", ".woff", ".ttf", ".woff2", ".webp",
+ ".inc",
+];
+
+static FILE_EXT_SET: LazyLock<HashSet<&'static str>> =
+ LazyLock::new(|| FILE_EXTENSIONS.iter().copied().collect());
+
#[derive(Deserialize)]
#[napi(object)]
pub struct FilterLinksCall {
@@ -20,6 +32,8 @@ pub struct FilterLinksCall {
pub allow_backward_crawling: bool,
pub ignore_robots_txt: bool,
pub robots_txt: String,
+ pub allow_external_content_links: bool,
+ pub allow_subdomains: bool,
}
#[derive(Serialize)]
@@ -29,6 +43,27 @@ pub struct FilterLinksResult {
pub denial_reasons: HashMap<String, String>,
}
+#[derive(Deserialize)]
+#[napi(object)]
+pub struct FilterUrlCall {
+ pub href: String,
+ pub url: String,
+ pub base_url: String,
+ pub excludes: Vec<String>,
+ pub ignore_robots_txt: bool,
+ pub robots_txt: String,
+ pub allow_external_content_links: bool,
+ pub allow_subdomains: bool,
+}
+
+#[derive(Serialize)]
+#[napi(object)]
+pub struct FilterUrlResult {
+ pub allowed: bool,
+ pub url: Option<String>,
+ pub denial_reason: Option<String>,
+}
+
#[derive(Serialize, Debug)]
#[napi(object)]
pub struct SitemapUrl {
@@ -75,150 +110,243 @@ pub struct SitemapProcessingResult {
pub total_count: u32,
}
-fn _is_file(url: &Url) -> bool {
- let file_extensions = vec![
- ".png", ".jpg", ".jpeg", ".gif", ".css", ".js", ".ico", ".svg", ".tiff", ".zip", ".exe",
- ".dmg", ".mp4", ".mp3", ".wav", ".pptx", ".xlsx", ".avi", ".flv", ".woff", ".ttf", ".woff2",
- ".webp", ".inc",
- ];
- let url_without_query = url.path().to_lowercase();
- file_extensions
- .iter()
- .any(|ext| url_without_query.ends_with(ext))
+const URL_PARSE_ERROR: &str = "URL_PARSE_ERROR";
+const DEPTH_LIMIT: &str = "DEPTH_LIMIT";
+const EXCLUDE_PATTERN: &str = "EXCLUDE_PATTERN";
+const INCLUDE_PATTERN: &str = "INCLUDE_PATTERN";
+const BACKWARD_CRAWLING: &str = "BACKWARD_CRAWLING";
+const ROBOTS_TXT: &str = "ROBOTS_TXT";
+const FILE_TYPE: &str = "FILE_TYPE";
+const SOCIAL_MEDIA: &str = "SOCIAL_MEDIA";
+const EXTERNAL_LINK: &str = "EXTERNAL_LINK";
+const SECTION_LINK: &str = "SECTION_LINK";
+
+#[inline]
+fn is_file(path: &str) -> bool {
+ if let Some(dot_pos) = path.rfind('.') {
+ let extension = &path[dot_pos..];
+ FILE_EXT_SET.contains(extension)
+ } else {
+ false
+ }
}
-fn _get_url_depth(path: &str) -> u32 {
+#[inline]
+fn get_url_depth(path: &str) -> u32 {
path
.split('/')
- .filter(|x| *x != "" && *x != "index.php" && *x != "index.html")
+ .filter(|segment| !segment.is_empty() && *segment != "index.php" && *segment != "index.html")
.count() as u32
}
-fn _filter_links(data: FilterLinksCall) -> std::result::Result<FilterLinksResult, String> {
- let mut denial_reasons = HashMap::new();
+#[inline]
+fn is_internal_link(url: &Url, base_url: &Url) -> bool {
+ let base_domain = base_url
+ .host_str()
+ .unwrap_or("")
+ .trim_start_matches("www.")
+ .trim();
+ let link_domain = url
+ .host_str()
+ .unwrap_or("")
+ .trim_start_matches("www.")
+ .trim();
+
+ link_domain == base_domain
+}
+
+#[inline]
+fn no_sections(url_str: &str) -> bool {
+ if !url_str.contains('#') {
+ return true;
+ }
+
+ // Check if the hash fragment looks like a route (contains forward slashes and has substantial content)
+ if let Some(hash_part) = url_str.split('#').nth(1) {
+ hash_part.len() > 1 && hash_part.contains('/')
+ } else {
+ false
+ }
+}
+
+#[inline]
+fn is_social_media_or_email(url_str: &str) -> bool {
+ const SOCIAL_MEDIA_OR_EMAIL: &[&str] = &[
+ "facebook.com",
+ "twitter.com",
+ "linkedin.com",
+ "instagram.com",
+ "pinterest.com",
+ "mailto:",
+ "github.com",
+ "calendly.com",
+ "discord.gg",
+ "discord.com",
+ ];
+
+ SOCIAL_MEDIA_OR_EMAIL
+ .iter()
+ .any(|domain| url_str.contains(domain))
+}
+
+#[inline]
+fn is_subdomain(url: &Url, base_url: &Url) -> bool {
+ match (url.host_str(), base_url.host_str()) {
+ (Some(link_host), Some(base_host)) => {
+ match (psl::domain_str(link_host), psl::domain_str(base_host)) {
+ (Some(link_domain), Some(base_domain)) => link_domain == base_domain,
+ _ => false,
+ }
+ }
+ _ => false,
+ }
+}
- let limit = data
- .limit
- .and_then(|x| if x < 0 { Some(0) } else { Some(x as usize) })
- .unwrap_or(usize::MAX);
+#[inline]
+fn is_external_main_page(url_str: &str) -> bool {
+ if let Ok(url) = Url::parse(url_str) {
+ let path_segments: Vec<&str> = url
+ .path_segments()
+ .map(|segments| segments.filter(|s| !s.is_empty()).collect())
+ .unwrap_or_default();
+ path_segments.is_empty()
+ } else {
+ false
+ }
+}
+fn _filter_links(data: FilterLinksCall) -> std::result::Result<FilterLinksResult, String> {
+ let limit = data.limit.map_or(usize::MAX, |x| x.max(0) as usize);
if limit == 0 {
return Ok(FilterLinksResult {
- links: Vec::with_capacity(0),
- denial_reasons,
+ links: Vec::new(),
+ denial_reasons: HashMap::new(),
});
}
- let base_url = Url::parse(&data.base_url).map_err(|e| format!("Base URL parse error: {}", e))?;
+ let base_url = Url::parse(&data.base_url).map_err(|e| format!("Base URL parse error: {e}"))?;
let initial_url =
- Url::parse(&data.initial_url).map_err(|e| format!("Initial URL parse error: {}", e))?;
+ Url::parse(&data.initial_url).map_err(|e| format!("Initial URL parse error: {e}"))?;
+ let initial_path = initial_url.path();
- let excludes_regex = data
+ let excludes_regex: Vec<Regex> = data
.excludes
.iter()
- .map(|exclude| Regex::new(exclude))
- .collect::<Vec<std::result::Result<Regex, regex::Error>>>();
- let excludes_regex = excludes_regex
- .into_iter()
- .filter_map(|x| x.ok())
- .collect::<Vec<Regex>>();
-
- let includes_regex = data
+ .filter_map(|e| Regex::new(e).ok())
+ .collect();
+ let includes_regex: Vec<Regex> = data
.includes
.iter()
- .map(|include| Regex::new(include))
- .collect::<Vec<std::result::Result<Regex, regex::Error>>>();
- let includes_regex = includes_regex
- .into_iter()
- .filter_map(|x| x.ok())
- .collect::<Vec<Regex>>();
-
- let links = data
- .links
- .into_iter()
- .filter(|link| {
- let url = match base_url.join(link) {
- Ok(x) => x,
- Err(_) => {
- denial_reasons.insert(link.clone(), "URL_PARSE_ERROR".to_string());
- return false;
- }
- };
+ .filter_map(|i| Regex::new(i).ok())
+ .collect();
+
+ let robot = if !data.ignore_robots_txt && !data.robots_txt.is_empty() {
+ Robot::new("FireCrawlAgent", data.robots_txt.as_bytes())
+ .ok()
+ .or_else(|| Robot::new("FirecrawlAgent", data.robots_txt.as_bytes()).ok())
+ } else {
+ None
+ };
+
+ let mut result_links = Vec::new();
+ let mut denial_reasons = HashMap::new();
+
+ for link in data.links {
+ if result_links.len() >= limit {
+ break;
+ }
+
+ let url = match base_url.join(&link) {
+ Ok(url) => url,
+ Err(_) => {
+ denial_reasons.insert(link, URL_PARSE_ERROR.to_string());
+ continue;
+ }
+ };
+
+ let path = url.path();
+ let url_str = url.as_str();
- let path = url.path();
- let depth = _get_url_depth(path);
- if depth > data.max_depth {
- denial_reasons.insert(link.clone(), "DEPTH_LIMIT".to_string());
- return false;
+ if get_url_depth(path) > data.max_depth {
+ denial_reasons.insert(link, DEPTH_LIMIT.to_string());
+ continue;
+ }
+
+ if is_file(path) {
+ denial_reasons.insert(link, FILE_TYPE.to_string());
+ continue;
+ }
+
+ if is_internal_link(&url, &base_url) {
+ // INTERNAL LINKS
+ if !no_sections(url_str) {
+ denial_reasons.insert(link, SECTION_LINK.to_string());
+ continue;
}
- let exinc_path = if data.regex_on_full_url {
- url.as_str()
+ if !data.allow_backward_crawling && !path.starts_with(initial_path) {
+ denial_reasons.insert(link, BACKWARD_CRAWLING.to_string());
+ continue;
+ }
+
+ let match_target = if data.regex_on_full_url {
+ url_str
} else {
- url.path()
+ path
};
- if !excludes_regex.is_empty()
- && excludes_regex
- .iter()
- .any(|regex| regex.is_match(exinc_path))
- {
- denial_reasons.insert(link.clone(), "EXCLUDE_PATTERN".to_string());
- return false;
+ if !excludes_regex.is_empty() && excludes_regex.iter().any(|r| r.is_match(match_target)) {
+ denial_reasons.insert(link, EXCLUDE_PATTERN.to_string());
+ continue;
}
- if !includes_regex.is_empty()
- && !includes_regex
- .iter()
- .any(|regex| regex.is_match(exinc_path))
- {
- denial_reasons.insert(link.clone(), "INCLUDE_PATTERN".to_string());
- return false;
+ if !includes_regex.is_empty() && !includes_regex.iter().any(|r| r.is_match(match_target)) {
+ denial_reasons.insert(link, INCLUDE_PATTERN.to_string());
+ continue;
}
- if !data.allow_backward_crawling {
- if !url.path().starts_with(initial_url.path()) {
- denial_reasons.insert(link.clone(), "BACKWARD_CRAWLING".to_string());
- return false;
+ if let Some(ref robot) = robot {
+ if !robot.allowed(url_str) {
+ denial_reasons.insert(link, ROBOTS_TXT.to_string());
+ continue;
}
}
- if !data.ignore_robots_txt {
- match Robot::new("FireCrawlAgent", data.robots_txt.as_bytes()) {
- Ok(robot) => {
- let allowed = robot.allowed(url.as_str());
- if !allowed {
- match Robot::new("FirecrawlAgent", data.robots_txt.as_bytes()) {
- Ok(robot_alt) => {
- let allowed_alt = robot_alt.allowed(url.as_str());
- if !allowed_alt {
- denial_reasons.insert(link.clone(), "ROBOTS_TXT".to_string());
- return false;
- }
- }
- Err(_) => {
- denial_reasons.insert(link.clone(), "ROBOTS_TXT".to_string());
- return false;
- }
- }
- }
- }
- Err(_) => {}
- }
+ result_links.push(link);
+ } else {
+ // EXTERNAL LINKS
+ if is_social_media_or_email(url_str) {
+ denial_reasons.insert(link, SOCIAL_MEDIA.to_string());
+ continue;
}
- if _is_file(&url) {
- denial_reasons.insert(link.clone(), "FILE_TYPE".to_string());
- return false;
+ if !excludes_regex.is_empty() && excludes_regex.iter().any(|r| r.is_match(url_str)) {
+ denial_reasons.insert(link, EXCLUDE_PATTERN.to_string());
+ continue;
}
- true
- })
- .take(limit)
- .collect::<Vec<_>>();
+ if is_internal_link(&initial_url, &base_url)
+ && data.allow_external_content_links
+ && !is_external_main_page(url_str)
+ {
+ result_links.push(link);
+ continue;
+ }
+
+ if data.allow_subdomains
+ && !is_social_media_or_email(url_str)
+ && is_subdomain(&url, &base_url)
+ {
+ result_links.push(link);
+ continue;
+ }
+
+ denial_reasons.insert(link, EXTERNAL_LINK.to_string());
+ }
+ }
Ok(FilterLinksResult {
- links,
+ links: result_links,
denial_reasons,
})
}
@@ -227,7 +355,169 @@ fn _filter_links(data: FilterLinksCall) -> std::result::Result<FilterLinksResult
#[napi]
pub fn filter_links(data: FilterLinksCall) -> Result<FilterLinksResult> {
_filter_links(data)
- .map_err(|e| Error::new(Status::GenericFailure, format!("Filter links error: {}", e)))
+ .map_err(|e| Error::new(Status::GenericFailure, format!("Filter links error: {e}")))
+}
+
+fn _filter_url(data: FilterUrlCall) -> std::result::Result<FilterUrlResult, String> {
+ let mut full_url = data.href.clone();
+
+ // Handle relative URLs
+ if !data.href.starts_with("http") {
+ match Url::parse(&data.url) {
+ Ok(base) => match base.join(&data.href) {
+ Ok(resolved) => full_url = resolved.to_string(),
+ Err(_) => {
+ return Ok(FilterUrlResult {
+ allowed: false,
+ url: None,
+ denial_reason: Some(URL_PARSE_ERROR.to_string()),
+ });
+ }
+ },
+ Err(_) => {
+ return Ok(FilterUrlResult {
+ allowed: false,
+ url: None,
+ denial_reason: Some(URL_PARSE_ERROR.to_string()),
+ });
+ }
+ }
+ }
+
+ let url = match Url::parse(&full_url) {
+ Ok(url) => url,
+ Err(_) => {
+ return Ok(FilterUrlResult {
+ allowed: false,
+ url: None,
+ denial_reason: Some(URL_PARSE_ERROR.to_string()),
+ });
+ }
+ };
+
+ let base_url = match Url::parse(&data.base_url) {
+ Ok(url) => url,
+ Err(_) => {
+ return Ok(FilterUrlResult {
+ allowed: false,
+ url: None,
+ denial_reason: Some(URL_PARSE_ERROR.to_string()),
+ });
+ }
+ };
+
+ let path = url.path();
+ let url_str = url.as_str();
+
+ let excludes_regex: Vec<Regex> = data
+ .excludes
+ .iter()
+ .filter_map(|e| Regex::new(e).ok())
+ .collect();
+
+ let robot = if !data.ignore_robots_txt && !data.robots_txt.is_empty() {
+ Robot::new("FireCrawlAgent", data.robots_txt.as_bytes())
+ .ok()
+ .or_else(|| Robot::new("FirecrawlAgent", data.robots_txt.as_bytes()).ok())
+ } else {
+ None
+ };
+
+ if is_internal_link(&url, &base_url) {
+ // INTERNAL LINKS
+ if !no_sections(url_str) {
+ return Ok(FilterUrlResult {
+ allowed: false,
+ url: None,
+ denial_reason: Some(SECTION_LINK.to_string()),
+ });
+ }
+
+ if !excludes_regex.is_empty() && excludes_regex.iter().any(|r| r.is_match(path)) {
+ return Ok(FilterUrlResult {
+ allowed: false,
+ url: None,
+ denial_reason: Some(EXCLUDE_PATTERN.to_string()),
+ });
+ }
+
+ if let Some(ref robot) = robot {
+ if !robot.allowed(url_str) {
+ return Ok(FilterUrlResult {
+ allowed: false,
+ url: None,
+ denial_reason: Some(ROBOTS_TXT.to_string()),
+ });
+ }
+ }
+
+ Ok(FilterUrlResult {
+ allowed: true,
+ url: Some(full_url),
+ denial_reason: None,
+ })
+ } else {
+ // EXTERNAL LINKS
+ if is_social_media_or_email(url_str) {
+ return Ok(FilterUrlResult {
+ allowed: false,
+ url: None,
+ denial_reason: Some(SOCIAL_MEDIA.to_string()),
+ });
+ }
+
+ if !excludes_regex.is_empty() && excludes_regex.iter().any(|r| r.is_match(url_str)) {
+ return Ok(FilterUrlResult {
+ allowed: false,
+ url: None,
+ denial_reason: Some(EXCLUDE_PATTERN.to_string()),
+ });
+ }
+
+ let context_url = match Url::parse(&data.url) {
+ Ok(url) => url,
+ Err(_) => {
+ return Ok(FilterUrlResult {
+ allowed: false,
+ url: None,
+ denial_reason: Some(URL_PARSE_ERROR.to_string()),
+ });
+ }
+ };
+
+ if is_internal_link(&context_url, &base_url)
+ && data.allow_external_content_links
+ && !is_external_main_page(url_str)
+ {
+ return Ok(FilterUrlResult {
+ allowed: true,
+ url: Some(full_url),
+ denial_reason: None,
+ });
+ }
+
+ if data.allow_subdomains && !is_social_media_or_email(url_str) && is_subdomain(&url, &base_url)
+ {
+ return Ok(FilterUrlResult {
+ allowed: true,
+ url: Some(full_url),
+ denial_reason: None,
+ });
+ }
+
+ Ok(FilterUrlResult {
+ allowed: false,
+ url: None,
+ denial_reason: Some(EXTERNAL_LINK.to_string()),
+ })
+ }
+}
+
+/// Filter a single URL based on crawling rules and constraints.
+#[napi]
+pub fn filter_url(data: FilterUrlCall) -> Result<FilterUrlResult> {
+ _filter_url(data)
+ .map_err(|e| Error::new(Status::GenericFailure, format!("Filter URL error: {e}")))
}
fn _parse_sitemap_xml(xml_content: &str) -> std::result::Result<ParsedSitemap, String> {
@@ -238,28 +528,24 @@ fn _parse_sitemap_xml(xml_content: &str) -> std::result::Result<ParsedSitemap, S
..Default::default()
},
)
- .map_err(|e| format!("XML parsing error: {}", e))?;
+ .map_err(|e| format!("XML parsing error: {e}"))?;
let root = doc.root_element();
match root.tag_name().name() {
"sitemapindex" => {
- let mut sitemaps = Vec::new();
-
- for sitemap_node in root
+ let sitemaps = root
.children()
.filter(|n| n.is_element() && n.tag_name().name() == "sitemap")
- {
- if let Some(loc_node) = sitemap_node
- .children()
- .find(|n| n.is_element() && n.tag_name().name() == "loc")
- {
- if let Some(loc_text) = loc_node.text() {
- sitemaps.push(SitemapEntry {
+ .filter_map(|sitemap_node| {
+ sitemap_node
+ .children()
+ .find(|n| n.is_element() && n.tag_name().name() == "loc")
+ .and_then(|loc_node| loc_node.text())
+ .map(|loc_text| SitemapEntry {
loc: vec![loc_text.to_string()],
- });
- }
- }
- }
+ })
+ })
+ .collect();
Ok(ParsedSitemap {
urlset: None,
@@ -267,23 +553,19 @@ fn _parse_sitemap_xml(xml_content: &str) -> std::result::Result<ParsedSitemap, S
})
}
"urlset" => {
- let mut urls = Vec::new();
-
- for url_node in root
+ let urls = root
.children()
.filter(|n| n.is_element() && n.tag_name().name() == "url")
- {
- if let Some(loc_node) = url_node
- .children()
- .find(|n| n.is_element() && n.tag_name().name() == "loc")
- {
- if let Some(loc_text) = loc_node.text() {
- urls.push(SitemapUrl {
+ .filter_map(|url_node| {
+ url_node
+ .children()
+ .find(|n| n.is_element() && n.tag_name().name() == "loc")
+ .and_then(|loc_node| loc_node.text())
+ .map(|loc_text| SitemapUrl {
loc: vec![loc_text.to_string()],
- });
- }
- }
- }
+ })
+ })
+ .collect();
Ok(ParsedSitemap {
urlset: Some(SitemapUrlset { url: urls }),
@@ -300,7 +582,7 @@ pub fn parse_sitemap_xml(xml_content: String) -> Result<ParsedSitemap> {
_parse_sitemap_xml(&xml_content).map_err(|e| {
Error::new(
Status::GenericFailure,
- format!("Parse sitemap XML error: {}", e),
+ format!("Parse sitemap XML error: {e}"),
)
})
}
@@ -324,12 +606,13 @@ fn _process_sitemap(xml_content: &str) -> std::result::Result<SitemapProcessingR
.collect();
if !sitemap_urls.is_empty() {
+ let count = sitemap_urls.len() as u32;
instructions.push(SitemapInstruction {
action: "recurse".to_string(),
- urls: sitemap_urls.clone(),
- count: sitemap_urls.len() as u32,
+ urls: sitemap_urls,
+ count,
});
- total_count += sitemap_urls.len() as u32;
+ total_count += count;
}
} else if let Some(urlset) = parsed.urlset {
let mut xml_sitemaps = Vec::new();
@@ -338,10 +621,12 @@ fn _process_sitemap(xml_content: &str) -> std::result::Result<SitemapProcessingR
for url_entry in urlset.url {
if !url_entry.loc.is_empty() {
let url = url_entry.loc[0].trim();
- if url.to_lowercase().ends_with(".xml") || url.to_lowercase().ends_with(".xml.gz") {
+ let url_lower = url.to_lowercase();
+ if url_lower.ends_with(".xml") || url_lower.ends_with(".xml.gz") {
xml_sitemaps.push(url.to_string());
} else if let Ok(parsed_url) = Url::parse(url) {
- if !_is_file(&parsed_url) {
+ let path_lower = parsed_url.path().to_lowercase();
+ if !is_file(&path_lower) {
valid_urls.push(url.to_string());
}
}
@@ -349,21 +634,23 @@ fn _process_sitemap(xml_content: &str) -> std::result::Result<SitemapProcessingR
}
if !xml_sitemaps.is_empty() {
+ let count = xml_sitemaps.len() as u32;
instructions.push(SitemapInstruction {
action: "recurse".to_string(),
- urls: xml_sitemaps.clone(),
- count: xml_sitemaps.len() as u32,
+ urls: xml_sitemaps,
+ count,
});
- total_count += xml_sitemaps.len() as u32;
+ total_count += count;
}
if !valid_urls.is_empty() {
+ let count = valid_urls.len() as u32;
instructions.push(SitemapInstruction {
action: "process".to_string(),
- urls: valid_urls.clone(),
- count: valid_urls.len() as u32,
+ urls: valid_urls,
+ count,
});
- total_count += valid_urls.len() as u32;
+ total_count += count;
}
}
@@ -379,7 +666,7 @@ pub fn process_sitemap(xml_content: String) -> Result<SitemapProcessingResult> {
_process_sitemap(&xml_content).map_err(|e| {
Error::new(
Status::GenericFailure,
- format!("Process sitemap error: {}", e),
+ format!("Process sitemap error: {e}"),
)
})
}
@@ -445,10 +732,7 @@ mod tests {
let result = _parse_sitemap_xml(xml_content);
assert!(result.is_err());
- assert!(result
- .unwrap_err()
- .to_string()
- .contains("Invalid sitemap format"));
+ assert!(result.unwrap_err().contains("Invalid sitemap format"));
}
#[test]
@@ -545,6 +829,8 @@ mod tests {
initial_url: "https://example.com".to_string(),
regex_on_full_url: false,
allow_backward_crawling: true,
+ allow_external_content_links: false,
+ allow_subdomains: false,
};
let result = _filter_links(data).unwrap();
@@ -577,6 +863,8 @@ mod tests {
initial_url: "https://example.com".to_string(),
regex_on_full_url: false,
allow_backward_crawling: true,
+ allow_external_content_links: false,
+ allow_subdomains: false,
};
let result = _filter_links(data);
@@ -604,6 +892,8 @@ mod tests {
initial_url: "https://example.com".to_string(),
regex_on_full_url: false,
allow_backward_crawling: true,
+ allow_external_content_links: false,
+ allow_subdomains: false,
};
let result = _filter_links(data);
@@ -629,6 +919,8 @@ mod tests {
initial_url: "https://example.com".to_string(),
regex_on_full_url: false,
allow_backward_crawling: true,
+ allow_external_content_links: false,
+ allow_subdomains: false,
};
let result = _filter_links(data);
@@ -637,4 +929,13 @@ mod tests {
assert_eq!(result.links.len(), 1);
assert_eq!(result.links[0], "https://example.com/test");
}
+
+ #[test]
+ fn test_is_file() {
+ assert!(is_file("test.png"));
+ assert!(is_file("script.js"));
+ assert!(is_file("style.css"));
+ assert!(!is_file("page"));
+ assert!(!is_file("directory/"));
+ }
}
diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.ts
index aaf966b1a..40d85b067 100644
--- a/apps/api/src/controllers/v2/map.ts
+++ b/apps/api/src/controllers/v2/map.ts
@@ -45,16 +45,19 @@ interface MapResult {
}
function dedupeMapDocumentArray(documents: MapDocument[]): MapDocument[] {
- let urlSet = new Set<string>(documents.map(x => x.url));
+ const urlMap = new Map<string, MapDocument>();
- let newDocuments: MapDocument[] = [
- ...Array.from(urlSet).map(x => {
- let localDocs = documents.filter(y => y.url === x);
- return localDocs.find(x => x.title !== undefined) || localDocs[0];
- }),
- ];
+ for (const doc of documents) {
+ const existing = urlMap.get(doc.url);
+
+ if (!existing) {
+ urlMap.set(doc.url, doc);
+ } else if (doc.title !== undefined && existing.title === undefined) {
+ urlMap.set(doc.url, doc);
+ }
+ }
- return newDocuments;
+ return Array.from(urlMap.values());
}
async function queryIndex(
@@ -116,6 +119,8 @@ async function getMapResults({
useIndex?: boolean;
location?: ScrapeOptions["location"];
}): Promise<MapResult> {
+ const functionStartTime = Date.now();
+
const id = uuidv4();
let mapResults: MapDocument[] = [];
const zeroDataRetention = flags?.forceZDR ?? false;
@@ -141,7 +146,9 @@ async function getMapResults({
try {
sc.robots = await crawler.getRobotsTxt(false, abort);
crawler.importRobotsTxt(sc.robots);
- } catch (_) {}
+ } catch (_) {
+ // Robots.txt fetch failed, continue without it
+ }
// If sitemapOnly is true, only get links from sitemap
if (crawlerOptions.sitemap === "only") {
@@ -159,6 +166,7 @@ async function getMapResults({
abort,
crawlerOptions.useMock,
);
+
if (sitemap > 0) {
mapResults = mapResults
.slice(1)
@@ -176,7 +184,6 @@ async function getMapResults({
}
} else {
let urlWithoutWww = url.replace("www.", "");
-
let mapUrl =
search && allowExternalLinks
? `${search} ${urlWithoutWww}`
@@ -198,7 +205,7 @@ async function getMapResults({
pagePromises = JSON.parse(cachedResult);
} else {
const fetchPage = async (page: number) => {
- return fireEngineMap(
+ return await fireEngineMap(
mapUrl,
{
numResults: resultsPerPage,
@@ -213,7 +220,6 @@ async function getMapResults({
);
}
- // Parallelize sitemap index query with search results
const [indexResults, searchResults] = await Promise.all([
queryIndex(url, limit, useIndex, includeSubdomains),
Promise.all(pagePromises),
@@ -232,8 +238,6 @@ async function getMapResults({
mapResults.push(...indexResults);
}
- // If sitemap is not ignored, fetch sitemap
- // This will attempt to find it in the index at first, or fetch a fresh one if it's older than 2 days
if (crawlerOptions.sitemap === "include") {
try {
await crawler.tryGetSitemap(
@@ -285,66 +289,64 @@ async function getMapResults({
const searchQuery = search.toLowerCase();
mapResults = performCosineSimilarityV2(mapResults, searchQuery);
}
+ }
- mapResults = mapResults
- .map(x => {
- try {
- return {
- ...x,
- url: checkAndUpdateURLForMap(
- x.url,
- crawlerOptions.ignoreQueryParameters ?? true,
- ).url.trim(),
- };
- } catch (_) {
- return null;
- }
- })
- .filter(x => x !== null) as MapDocument[];
-
- // allows for subdomains to be included
- mapResults = mapResults.filter(x => isSameDomain(x.url, url));
-
- // if includeSubdomains is false, filter out subdomains
- if (!includeSubdomains) {
- mapResults = mapResults.filter(x => isSameSubdomain(x.url, url));
- }
-
- // Filter by path if enabled
- if (filterByPath && !allowExternalLinks) {
+ mapResults = mapResults
+ .map(x => {
try {
- const urlObj = new URL(url);
- const urlPath = urlObj.pathname;
- // Only apply path filtering if the URL has a significant path (not just '/' or empty)
- // This means we only filter by path if the user has not selected a root domain
- if (urlPath && urlPath !== "/" && urlPath.length > 1) {
- mapResults = mapResults.filter(x => {
- try {
- const linkObj = new URL(x.url);
- return linkObj.pathname.startsWith(urlPath);
- } catch (e) {
- return false;
- }
- });
- }
- } catch (e) {
- // If URL parsing fails, continue without path filtering
- logger.warn(`Failed to parse URL for path filtering: ${url}`, {
- error: e,
+ return {
+ ...x,
+ url: checkAndUpdateURLForMap(
+ x.url,
+ crawlerOptions.ignoreQueryParameters ?? true,
+ ).url.trim(),
+ };
+ } catch (_) {
+ return null;
+ }
+ })
+ .filter(x => x !== null) as MapDocument[];
+
+ mapResults = mapResults.filter(x => isSameDomain(x.url, url));
+
+ if (!includeSubdomains) {
+ mapResults = mapResults.filter(x => isSameSubdomain(x.url, url));
+ }
+
+ if (filterByPath && !allowExternalLinks) {
+ try {
+ const urlObj = new URL(url);
+ const urlPath = urlObj.pathname;
+ // Only apply path filtering if the URL has a significant path (not just '/' or empty)
+ // This means we only filter by path if the user has not selected a root domain
+ if (urlPath && urlPath !== "/" && urlPath.length > 1) {
+ mapResults = mapResults.filter(x => {
+ try {
+ const linkObj = new URL(x.url);
+ return linkObj.pathname.startsWith(urlPath);
+ } catch (e) {
+ return false;
+ }
});
}
+ } catch (e) {
+ // If URL parsing fails, continue without path filtering
+ logger.warn(`Failed to parse URL for path filtering: ${url}`, {
+ error: e,
+ });
}
-
- mapResults = dedupeMapDocumentArray(mapResults);
}
+ mapResults = dedupeMapDocumentArray(mapResults);
mapResults = mapResults.slice(0, limit);
+ const totalTimeMs = Date.now() - functionStartTime;
+
return {
success: true,
mapResults,
job_id: id,
- time_taken: (new Date().getTime() - Date.now()) / 1000,
+ time_taken: totalTimeMs,
};
}
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index c58a148c8..b86cbf355 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -15,7 +15,7 @@ import {
} from "../../lib/robots-txt";
import { ScrapeJobTimeoutError } from "../../lib/error";
import { ScrapeOptions } from "../../controllers/v2/types";
-import { filterLinks } from "@mendable/firecrawl-rs";
+import { filterLinks, filterUrl } from "@mendable/firecrawl-rs";
export const SITEMAP_LIMIT = 100;
@@ -175,6 +175,8 @@ export class WebCrawler {
allowBackwardCrawling: this.allowBackwardCrawling,
ignoreRobotsTxt: this.ignoreRobotsTxt,
robotsTxt: this.robotsTxt,
+ allowExternalContentLinks: this.allowExternalContentLinks,
+ allowSubdomains: this.allowSubdomains,
});
const fancyDenialReasons = new Map<string, string>();
@@ -427,32 +429,36 @@ export class WebCrawler {
return await urlsHandler(urls);
} else {
let filteredLinksResult = await this.filterLinks(
- [...new Set(urls)].filter(
- x => this.filterURL(x, this.initialUrl).allowed,
- ),
+ [...new Set(urls)],
leftOfLimit,
this.maxCrawledDepth,
fromMap,
);
let filteredLinks = filteredLinksResult.links;
leftOfLimit -= filteredLinks.length;
- let uniqueURLs: string[] = [];
- for (const url of filteredLinks) {
- if (
- await redisEvictConnection.sadd(
- "sitemap:" + this.jobId + ":links",
- normalizeUrl(url),
- )
- ) {
- uniqueURLs.push(url);
- }
- }
+ const pipeline = redisEvictConnection.pipeline();
+
+ const normalizedUrls = filteredLinks.map(url => normalizeUrl(url));
+ normalizedUrls.forEach(normalizedUrl => {
+ pipeline.sadd("sitemap:" + this.jobId + ":links", normalizedUrl);
+ });
+
+ const results = await pipeline.exec();
+
+ const uniqueURLs = filteredLinks.filter(
+ (_, index) =>
+ results &&
+ results[index] &&
+ !results[index][0] &&
+ results[index][1] === 1,
+ );
await redisEvictConnection.expire(
"sitemap:" + this.jobId + ":links",
3600,
"NX",
);
+
if (uniqueURLs.length > 0) {
return await urlsHandler(uniqueURLs);
}
@@ -528,83 +534,24 @@ export class WebCrawler {
}
}
- public filterURL(href: string, url: string): FilterResult {
- let fullUrl = href;
- if (!href.startsWith("http")) {
- try {
- fullUrl = new URL(href, url).toString();
- } catch (_) {
- return { allowed: false, denialReason: DenialReason.URL_PARSE_ERROR };
- }
- }
- let urlObj;
- try {
- urlObj = new URL(fullUrl);
- } catch (_) {
- return { allowed: false, denialReason: DenialReason.URL_PARSE_ERROR };
- }
- const path = urlObj.pathname;
-
- if (this.isInternalLink(fullUrl)) {
- // INTERNAL LINKS
- if (!this.noSections(fullUrl)) {
- return { allowed: false, denialReason: DenialReason.SECTION_LINK };
- }
-
- if (this.matchesExcludes(path)) {
- return { allowed: false, denialReason: DenialReason.EXCLUDE_PATTERN };
- }
-
- if (!this.isRobotsAllowed(fullUrl, this.ignoreRobotsTxt)) {
- (async () => {
- await redisEvictConnection.sadd(
- "crawl:" + this.jobId + ":robots_blocked",
- fullUrl,
- );
- await redisEvictConnection.expire(
- "crawl:" + this.jobId + ":robots_blocked",
- 24 * 60 * 60,
- );
- })();
- return { allowed: false, denialReason: DenialReason.ROBOTS_TXT };
- }
-
- return { allowed: true, url: fullUrl };
- } else {
- // EXTERNAL LINKS
- if (this.isSocialMediaOrEmail(fullUrl)) {
- return { allowed: false, denialReason: DenialReason.SOCIAL_MEDIA };
- }
-
- if (this.matchesExcludes(fullUrl, true)) {
- return { allowed: false, denialReason: DenialReason.EXCLUDE_PATTERN };
- }
-
- if (
- this.isInternalLink(url) &&
- this.allowExternalContentLinks &&
- !this.isExternalMainPage(fullUrl)
- ) {
- return { allowed: true, url: fullUrl };
- }
-
- if (
- this.allowSubdomains &&
- !this.isSocialMediaOrEmail(fullUrl) &&
- this.isSubdomain(fullUrl)
- ) {
- return { allowed: true, url: fullUrl };
- }
-
- return { allowed: false, denialReason: DenialReason.EXTERNAL_LINK };
- }
+ public async filterURL(href: string, url: string): Promise<FilterResult> {
+ return await filterUrl({
+ href: href,
+ url: url,
+ baseUrl: this.baseUrl,
+ excludes: this.excludes,
+ ignoreRobotsTxt: this.ignoreRobotsTxt,
+ robotsTxt: this.robotsTxt,
+ allowExternalContentLinks: this.allowExternalContentLinks,
+ allowSubdomains: this.allowSubdomains,
+ });
}
private async extractLinksFromHTMLRust(html: string, url: string) {
const links = await extractLinks(html);
const filteredLinks: string[] = [];
for (const link of links) {
- const filterResult = this.filterURL(link, url);
+ const filterResult = await this.filterURL(link, url);
if (filterResult.allowed && filterResult.url) {
filteredLinks.push(filterResult.url);
}
@@ -612,32 +559,37 @@ export class WebCrawler {
return filteredLinks;
}
- private extractLinksFromHTMLCheerio(html: string, url: string) {
+ private async extractLinksFromHTMLCheerio(html: string, url: string) {
let links: string[] = [];
const $ = load(html);
- $("a").each((_, element) => {
+ for (let i = 0; i < $("a").length; i++) {
+ const element = $("a")[i];
let href = $(element).attr("href");
if (href) {
if (href.match(/^https?:\/[^\/]/)) {
href = href.replace(/^https?:\//, "$&/");
}
- const filterResult = this.filterURL(href, url);
+ const filterResult = await this.filterURL(href, url);
if (filterResult.allowed && filterResult.url) {
links.push(filterResult.url);
}
}
- });
+ }
// Extract links from iframes with inline src
- $("iframe").each((_, element) => {
+ for (let i = 0; i < $("iframe").length; i++) {
+ const element = $("iframe")[i];
const src = $(element).attr("src");
if (src && src.startsWith("data:text/html")) {
const iframeHtml = decodeURIComponent(src.split(",")[1]);
- const iframeLinks = this.extractLinksFromHTMLCheerio(iframeHtml, url);
+ const iframeLinks = await this.extractLinksFromHTMLCheerio(
+ iframeHtml,
+ url,
+ );
links = links.concat(iframeLinks);
}
- });
+ }
return links;
}
@@ -668,7 +620,7 @@ export class WebCrawler {
);
}
- return this.extractLinksFromHTMLCheerio(html, url);
+ return await this.extractLinksFromHTMLCheerio(html, url);
}
private isRobotsAllowed(
@@ -678,94 +630,6 @@ export class WebCrawler {
return ignoreRobotsTxt ? true : isUrlAllowedByRobots(url, this.robots);
}
- private matchesExcludes(url: string, onlyDomains: boolean = false): boolean {
- return this.excludes.some(pattern => {
- if (onlyDomains) return this.matchesExcludesExternalDomains(url);
-
- return this.excludes.some(pattern => new RegExp(pattern).test(url));
- });
- }
-
- // supported formats: "example.com/blog", "https://example.com", "blog.example.com", "example.com"
- private matchesExcludesExternalDomains(url: string) {
- try {
- const urlObj = new URL(url);
- const hostname = urlObj.hostname;
- const pathname = urlObj.pathname;
-
- for (let domain of this.excludes) {
- let domainObj = new URL("http://" + domain.replace(/^https?:\/\//, ""));
- let domainHostname = domainObj.hostname;
- let domainPathname = domainObj.pathname;
-
- if (
- hostname === domainHostname ||
- hostname.endsWith(`.${domainHostname}`)
- ) {
- if (pathname.startsWith(domainPathname)) {
- return true;
- }
- }
- }
- return false;
- } catch (e) {
- return false;
- }
- }
-
- private isExternalMainPage(url: string): boolean {
- return !Boolean(
- url
- .split("/")
- .slice(3)
- .filter(subArray => subArray.length > 0).length,
- );
- }
-
- private noSections(link: string): boolean {
- // Allow URLs with hash fragments that represent actual routes/pages (like SPAs)
- // but block simple anchor links within the same page
- if (!link.includes("#")) {
- return true;
- }
-
- // Check if the hash fragment looks like a route (contains forward slashes and has substantial content)
- const hashPart = link.split("#")[1];
- if (hashPart && hashPart.length > 1 && hashPart.includes("/")) {
- return true;
- }
-
- return false;
- }
-
- private isInternalLink(link: string): boolean {
- const urlObj = new URL(link, this.baseUrl);
- const baseDomain = new URL(this.baseUrl).hostname
- .replace(/^www\./, "")
- .trim();
- const linkDomain = urlObj.hostname.replace(/^www\./, "").trim();
-
- return linkDomain === baseDomain;
- }
-
- private isSubdomain(link: string): boolean {
- try {
- const linkUrl = new URL(link, this.baseUrl);
- const baseUrl = new URL(this.baseUrl);
-
- const linkParsed = psl.parse(linkUrl.hostname);
- const baseParsed = psl.parse(baseUrl.hostname);
-
- if (!linkParsed?.domain || !baseParsed?.domain) {
- return false;
- }
-
- return linkParsed.domain === baseParsed.domain;
- } catch (error) {
- return false;
- }
- }
-
public isFile(url: string): boolean {
const fileExtensions = [
".png",
@@ -809,22 +673,6 @@ export class WebCrawler {
}
}
- private isSocialMediaOrEmail(url: string): boolean {
- const socialMediaOrEmail = [
- "facebook.com",
- "twitter.com",
- "linkedin.com",
- "instagram.com",
- "pinterest.com",
- "mailto:",
- "github.com",
- "calendly.com",
- "discord.gg",
- "discord.com",
- ];
- return socialMediaOrEmail.some(ext => url.includes(ext));
- }
-
private async tryFetchSitemapLinks(
url: string,
urlsHandler: (urls: string[]) => unknown,
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index 72dff9795..02efe206c 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -72,6 +72,7 @@ import {
ScrapeJobSingleUrls,
} from "../../types";
import { scrapeSitemap } from "../../scraper/crawler/sitemap";
+import { filterUrl } from "@mendable/firecrawl-rs";
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
@@ -254,7 +255,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
sc,
(await getACUCTeam(job.data.team_id))?.flags ?? null,
);
- const filterResult = crawler.filterURL(
+ const filterResult = await crawler.filterURL(
doc.metadata.url,
doc.metadata.sourceURL,
);
@@ -696,7 +697,7 @@ async function kickoffGetIndexLinks(
);
const validIndexLinksResult = await crawler.filterLinks(
- index.filter(x => crawler.filterURL(x, trimmedURL.href).allowed),
+ index,
sc.crawlerOptions.limit ?? 10000,
sc.crawlerOptions.maxDepth ?? 10,
false,
@@ -955,9 +956,7 @@ async function processKickoffSitemapJob(job: NuQJob<ScrapeJobKickoffSitemap>) {
const passingURLs = (
await crawler.filterLinks(
- [...new Set(results.urls.map(x => x.href))].filter(
- url => crawler.filterURL(url, sc.originUrl!).allowed,
- ),
+ results.urls.map(x => x.href),
Infinity,
sc.crawlerOptions.maxDepth ?? 10,
false,