Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Merge pull request #2278 from firecrawl/abi/eng-3707-bug-when-using-both-include-paths-and-allow-subdomain

Abimael Martell6c7f556bb602
diff --git a/apps/api/native/src/crawler.rs b/apps/api/native/src/crawler.rs
index 399d4e21c..81aef2f2d 100644
--- a/apps/api/native/src/crawler.rs
+++ b/apps/api/native/src/crawler.rs
@@ -337,6 +337,14 @@ fn _filter_links(data: FilterLinksCall) -> std::result::Result<FilterLinksResult
&& !is_social_media_or_email(url_str)
&& is_subdomain(&url, &base_url)
{
+ // When allowing subdomains, still honor include patterns
+ let match_target = if data.regex_on_full_url { url_str } else { path };
+ if !includes_regex.is_empty()
+ && !includes_regex.iter().any(|r| r.is_match(match_target))
+ {
+ denial_reasons.insert(link, INCLUDE_PATTERN.to_string());
+ continue;
+ }
result_links.push(link);
continue;
}
@@ -930,6 +938,55 @@ mod tests {
assert_eq!(result.links[0], "https://example.com/test");
}
+ #[test]
+ fn test_filter_links_allow_subdomains_with_include_paths() {
+ let data = FilterLinksCall {
+ links: vec![
+ "https://sub.example.com/pricing".to_string(),
+ "https://sub.example.com/blog".to_string(),
+ "https://other.example.com/pricing".to_string(),
+ "https://example.com/pricing".to_string(),
+ ],
+ limit: Some(10),
+ includes: vec!["^/pricing$".to_string()],
+ excludes: vec![],
+ ignore_robots_txt: true,
+ robots_txt: "".to_string(),
+ max_depth: 10,
+ base_url: "https://example.com".to_string(),
+ initial_url: "https://example.com".to_string(),
+ regex_on_full_url: false,
+ allow_backward_crawling: true,
+ allow_external_content_links: false,
+ allow_subdomains: true,
+ };
+
+ let result = _filter_links(data).unwrap();
+ // Should include only paths matching include on base or subdomains
+ assert_eq!(result.links.len(), 3);
+ assert!(result
+ .links
+ .contains(&"https://example.com/pricing".to_string()));
+ assert!(result
+ .links
+ .contains(&"https://sub.example.com/pricing".to_string()));
+ assert!(result
+ .links
+ .contains(&"https://other.example.com/pricing".to_string()));
+ // And should exclude blog due to includePaths
+ assert!(result
+ .denial_reasons
+ .get("https://sub.example.com/blog")
+ .is_some());
+ assert_eq!(
+ result
+ .denial_reasons
+ .get("https://sub.example.com/blog")
+ .unwrap(),
+ "INCLUDE_PATTERN"
+ );
+ }
+
#[test]
fn test_is_file() {
assert!(is_file("test.png"));
diff --git a/apps/api/src/scraper/WebScraper/__tests__/crawler.test.ts b/apps/api/src/scraper/WebScraper/__tests__/crawler.test.ts
index 33b3d74a5..0f8a110c2 100644
--- a/apps/api/src/scraper/WebScraper/__tests__/crawler.test.ts
+++ b/apps/api/src/scraper/WebScraper/__tests__/crawler.test.ts
@@ -73,4 +73,83 @@ describe("WebCrawler", () => {
expect(filteredLinks.links.length).toBe(limit); // Check if the number of results respects the limit
expect(filteredLinks.links).toEqual([initialUrl, initialUrl + "/page1"]);
});
+
+ it("should filter subdomain URLs with includePaths when allowSubdomains is true", async () => {
+ const initialUrl = "https://example.com";
+
+ crawler = new WebCrawler({
+ jobId: "TEST",
+ initialUrl: initialUrl,
+ includes: ["^/pricing$"], // Only allow /pricing path
+ excludes: [],
+ limit: 10,
+ maxCrawledDepth: 10,
+ allowSubdomains: true,
+ });
+
+ const linksToFilter = [
+ "https://example.com/pricing", // Should pass: base domain + /pricing
+ "https://example.com/blog", // Should fail: base domain but wrong path
+ "https://sub.example.com/pricing", // Should pass: subdomain + /pricing
+ "https://sub.example.com/about", // Should fail: subdomain but wrong path
+ "https://other.example.com/pricing", // Should pass: subdomain + /pricing
+ "https://other.example.com/contact", // Should fail: subdomain but wrong path
+ ];
+
+ const filteredLinks = await crawler["filterLinks"](linksToFilter, 10, 10);
+
+ expect(filteredLinks.links.length).toBe(3);
+ expect(filteredLinks.links).toContain("https://example.com/pricing");
+ expect(filteredLinks.links).toContain("https://sub.example.com/pricing");
+ expect(filteredLinks.links).toContain("https://other.example.com/pricing");
+
+ // Verify denied links
+ expect(filteredLinks.denialReasons.has("https://example.com/blog")).toBe(
+ true,
+ );
+ expect(
+ filteredLinks.denialReasons.has("https://sub.example.com/about"),
+ ).toBe(true);
+ expect(
+ filteredLinks.denialReasons.has("https://other.example.com/contact"),
+ ).toBe(true);
+ });
+
+ it("should filter subdomain URLs with includePaths using regexOnFullURL", async () => {
+ const initialUrl = "https://example.com";
+
+ crawler = new WebCrawler({
+ jobId: "TEST",
+ initialUrl: initialUrl,
+ includes: ["^https://([a-z0-9-]+\\.)?example\\.com/pricing$"], // Full URL pattern
+ excludes: [],
+ limit: 10,
+ maxCrawledDepth: 10,
+ allowSubdomains: true,
+ regexOnFullURL: true,
+ });
+
+ const linksToFilter = [
+ "https://example.com/pricing", // Should pass: matches pattern
+ "https://example.com/pricing/details", // Should fail: doesn't match exact pattern
+ "https://sub.example.com/pricing", // Should pass: subdomain + /pricing
+ "https://api.example.com/pricing", // Should pass: subdomain + /pricing
+ "https://sub.example.com/blog", // Should fail: wrong path
+ ];
+
+ const filteredLinks = await crawler["filterLinks"](linksToFilter, 10, 10);
+
+ expect(filteredLinks.links.length).toBe(3);
+ expect(filteredLinks.links).toContain("https://example.com/pricing");
+ expect(filteredLinks.links).toContain("https://sub.example.com/pricing");
+ expect(filteredLinks.links).toContain("https://api.example.com/pricing");
+
+ // Verify denied links
+ expect(
+ filteredLinks.denialReasons.has("https://example.com/pricing/details"),
+ ).toBe(true);
+ expect(
+ filteredLinks.denialReasons.has("https://sub.example.com/blog"),
+ ).toBe(true);
+ });
});