Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Merge pull request #2278 from firecrawl/abi/eng-3707-bug-when-using-both-include-paths-and-allow-subdomain
Abimael Martell6c7f556bb602
diff --git a/apps/api/native/src/crawler.rs b/apps/api/native/src/crawler.rsindex 399d4e21c..81aef2f2d 100644--- a/apps/api/native/src/crawler.rs+++ b/apps/api/native/src/crawler.rs@@ -337,6 +337,14 @@ fn _filter_links(data: FilterLinksCall) -> std::result::Result<FilterLinksResult&& !is_social_media_or_email(url_str)&& is_subdomain(&url, &base_url){+ // When allowing subdomains, still honor include patterns+ let match_target = if data.regex_on_full_url { url_str } else { path };+ if !includes_regex.is_empty()+ && !includes_regex.iter().any(|r| r.is_match(match_target))+ {+ denial_reasons.insert(link, INCLUDE_PATTERN.to_string());+ continue;+ }result_links.push(link);continue;}@@ -930,6 +938,55 @@ mod tests {assert_eq!(result.links[0], "https://example.com/test");}+ #[test]+ fn test_filter_links_allow_subdomains_with_include_paths() {+ let data = FilterLinksCall {+ links: vec![+ "https://sub.example.com/pricing".to_string(),+ "https://sub.example.com/blog".to_string(),+ "https://other.example.com/pricing".to_string(),+ "https://example.com/pricing".to_string(),+ ],+ limit: Some(10),+ includes: vec!["^/pricing$".to_string()],+ excludes: vec![],+ ignore_robots_txt: true,+ robots_txt: "".to_string(),+ max_depth: 10,+ base_url: "https://example.com".to_string(),+ initial_url: "https://example.com".to_string(),+ regex_on_full_url: false,+ allow_backward_crawling: true,+ allow_external_content_links: false,+ allow_subdomains: true,+ };++ let result = _filter_links(data).unwrap();+ // Should include only paths matching include on base or subdomains+ assert_eq!(result.links.len(), 3);+ assert!(result+ .links+ .contains(&"https://example.com/pricing".to_string()));+ assert!(result+ .links+ .contains(&"https://sub.example.com/pricing".to_string()));+ assert!(result+ .links+ .contains(&"https://other.example.com/pricing".to_string()));+ // And should exclude blog due to includePaths+ assert!(result+ .denial_reasons+ .get("https://sub.example.com/blog")+ .is_some());+ assert_eq!(+ result+ .denial_reasons+ .get("https://sub.example.com/blog")+ .unwrap(),+ "INCLUDE_PATTERN"+ );+ }+#[test]fn test_is_file() {assert!(is_file("test.png"));diff --git a/apps/api/src/scraper/WebScraper/__tests__/crawler.test.ts b/apps/api/src/scraper/WebScraper/__tests__/crawler.test.tsindex 33b3d74a5..0f8a110c2 100644--- a/apps/api/src/scraper/WebScraper/__tests__/crawler.test.ts+++ b/apps/api/src/scraper/WebScraper/__tests__/crawler.test.ts@@ -73,4 +73,83 @@ describe("WebCrawler", () => {expect(filteredLinks.links.length).toBe(limit); // Check if the number of results respects the limitexpect(filteredLinks.links).toEqual([initialUrl, initialUrl + "/page1"]);});++ it("should filter subdomain URLs with includePaths when allowSubdomains is true", async () => {+ const initialUrl = "https://example.com";++ crawler = new WebCrawler({+ jobId: "TEST",+ initialUrl: initialUrl,+ includes: ["^/pricing$"], // Only allow /pricing path+ excludes: [],+ limit: 10,+ maxCrawledDepth: 10,+ allowSubdomains: true,+ });++ const linksToFilter = [+ "https://example.com/pricing", // Should pass: base domain + /pricing+ "https://example.com/blog", // Should fail: base domain but wrong path+ "https://sub.example.com/pricing", // Should pass: subdomain + /pricing+ "https://sub.example.com/about", // Should fail: subdomain but wrong path+ "https://other.example.com/pricing", // Should pass: subdomain + /pricing+ "https://other.example.com/contact", // Should fail: subdomain but wrong path+ ];++ const filteredLinks = await crawler["filterLinks"](linksToFilter, 10, 10);++ expect(filteredLinks.links.length).toBe(3);+ expect(filteredLinks.links).toContain("https://example.com/pricing");+ expect(filteredLinks.links).toContain("https://sub.example.com/pricing");+ expect(filteredLinks.links).toContain("https://other.example.com/pricing");++ // Verify denied links+ expect(filteredLinks.denialReasons.has("https://example.com/blog")).toBe(+ true,+ );+ expect(+ filteredLinks.denialReasons.has("https://sub.example.com/about"),+ ).toBe(true);+ expect(+ filteredLinks.denialReasons.has("https://other.example.com/contact"),+ ).toBe(true);+ });++ it("should filter subdomain URLs with includePaths using regexOnFullURL", async () => {+ const initialUrl = "https://example.com";++ crawler = new WebCrawler({+ jobId: "TEST",+ initialUrl: initialUrl,+ includes: ["^https://([a-z0-9-]+\\.)?example\\.com/pricing$"], // Full URL pattern+ excludes: [],+ limit: 10,+ maxCrawledDepth: 10,+ allowSubdomains: true,+ regexOnFullURL: true,+ });++ const linksToFilter = [+ "https://example.com/pricing", // Should pass: matches pattern+ "https://example.com/pricing/details", // Should fail: doesn't match exact pattern+ "https://sub.example.com/pricing", // Should pass: subdomain + /pricing+ "https://api.example.com/pricing", // Should pass: subdomain + /pricing+ "https://sub.example.com/blog", // Should fail: wrong path+ ];++ const filteredLinks = await crawler["filterLinks"](linksToFilter, 10, 10);++ expect(filteredLinks.links.length).toBe(3);+ expect(filteredLinks.links).toContain("https://example.com/pricing");+ expect(filteredLinks.links).toContain("https://sub.example.com/pricing");+ expect(filteredLinks.links).toContain("https://api.example.com/pricing");++ // Verify denied links+ expect(+ filteredLinks.denialReasons.has("https://example.com/pricing/details"),+ ).toBe(true);+ expect(+ filteredLinks.denialReasons.has("https://sub.example.com/blog"),+ ).toBe(true);+ });});