Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): follow external link redirects within their own domain
firecrawl-spring[bot]1dadb1b1e9cf
diff --git a/apps/api/native/src/crawler.rs b/apps/api/native/src/crawler.rsindex 66023c2a3..9cf64da24 100644--- a/apps/api/native/src/crawler.rs+++ b/apps/api/native/src/crawler.rs@@ -581,9 +581,13 @@ fn _filter_url(data: FilterUrlCall) -> std::result::Result<FilterUrlResult, Stri}};- if is_internal_link(&context_url, &base_url)- && data.allow_external_content_links+ // Allow an external destination when external content links are enabled and+ // it is not an external site's homepage. Two cases qualify: the link was+ // found on an in-scope page, or an already-admitted external link redirected+ // within its own domain (i.e. resolving to its canonical URL).+ if data.allow_external_content_links&& !is_external_main_page(url_str)+ && (is_internal_link(&context_url, &base_url) || is_internal_link(&url, &context_url)){return Ok(FilterUrlResult {allowed: true,@@ -1194,4 +1198,74 @@ mod tests {assert!(!is_file("page"));assert!(!is_file("directory/"));}++ fn filter_url_call(href: &str, url: &str, base_url: &str) -> FilterUrlCall {+ FilterUrlCall {+ href: href.to_string(),+ url: url.to_string(),+ base_url: base_url.to_string(),+ excludes: vec![],+ ignore_robots_txt: true,+ robots_txt: "".to_string(),+ robots_user_agent: None,+ allow_external_content_links: true,+ allow_subdomains: false,+ }+ }++ // A discovered external link that redirects within its own domain (the+ // redirect target's URL and its source URL are both external to the crawl+ // seed) must be allowed under allowExternalLinks. Regression test for #4315.+ #[test]+ fn test_filter_url_allows_external_link_redirect_same_domain() {+ let result = _filter_url(filter_url_call(+ "http://www.iana.org/help/example-domains",+ "https://iana.org/domains/example",+ "https://example.org",+ ))+ .unwrap();+ assert!(result.allowed);+ assert!(result.denial_reason.is_none());+ }++ // Same as above but the redirect keeps the exact hostname, only changing the+ // path (the london.gov.uk case from #4315).+ #[test]+ fn test_filter_url_allows_external_link_redirect_same_host_path_change() {+ let result = _filter_url(filter_url_call(+ "https://www.london.gov.uk/programmes-strategies/planning/london-plan",+ "https://www.london.gov.uk/what-we-do/planning/london-plan",+ "https://example.org",+ ))+ .unwrap();+ assert!(result.allowed);+ }++ // An in-scope URL that redirects to an unrelated external homepage stays+ // denied — the external-main-page exclusion is intentional (#4316).+ #[test]+ fn test_filter_url_denies_redirect_to_external_homepage() {+ let result = _filter_url(filter_url_call(+ "https://www.peoplefirstinfo.org.uk/",+ "https://www.westminster.gov.uk/node/21229",+ "https://www.westminster.gov.uk",+ ))+ .unwrap();+ assert!(!result.allowed);+ assert_eq!(result.denial_reason.unwrap(), "EXTERNAL_LINK");+ }++ // A redirect that hops to an unrelated external domain (not the source's own+ // domain, not in scope) remains denied.+ #[test]+ fn test_filter_url_denies_redirect_to_unrelated_external_domain() {+ let result = _filter_url(filter_url_call(+ "https://unrelated.com/article",+ "https://iana.org/domains/example",+ "https://example.org",+ ))+ .unwrap();+ assert!(!result.allowed);+ assert_eq!(result.denial_reason.unwrap(), "EXTERNAL_LINK");+ }}diff --git a/apps/api/src/__tests__/snips/v2/crawl.test.ts b/apps/api/src/__tests__/snips/v2/crawl.test.tsindex e6ebc0e2a..f41a21353 100644--- a/apps/api/src/__tests__/snips/v2/crawl.test.ts+++ b/apps/api/src/__tests__/snips/v2/crawl.test.ts@@ -447,6 +447,41 @@ describe("Crawl tests", () => {5 * scrapeTimeout,);+ // Regression for #4315: with allowExternalLinks, a discovered external link+ // that redirects within its own domain must be followed and scraped, not+ // rejected as EXTERNAL_LINK. example.org links to iana.org/domains/example,+ // which redirects to www.iana.org/help/example-domains.+ concurrentIf(!process.env.TEST_SUITE_SELF_HOSTED)(+ "allowExternalLinks follows a redirecting external link",+ async () => {+ const res = await crawl(+ {+ url: "https://example.org/",+ limit: 2,+ maxDiscoveryDepth: 1,+ allowExternalLinks: true,+ sitemap: "skip",+ },+ identity,+ );++ expect(res.success).toBe(true);+ if (res.success) {+ const scrapedIana = res.data.some(page => {+ try {+ const host = new URL(page.metadata.url ?? page.metadata.sourceURL!)+ .hostname;+ return host === "iana.org" || host.endsWith(".iana.org");+ } catch {+ return false;+ }+ });+ expect(scrapedIana).toBe(true);+ }+ },+ 5 * scrapeTimeout,+ );+describeIf(TEST_PRODUCTION || (HAS_AI && ALLOW_TEST_SUITE_WEBSITE))("Crawl API with Prompt",() => {