Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): follow external link redirects within their own domain

firecrawl-spring[bot]1dadb1b1e9cf
diff --git a/apps/api/native/src/crawler.rs b/apps/api/native/src/crawler.rs
index 66023c2a3..9cf64da24 100644
--- a/apps/api/native/src/crawler.rs
+++ b/apps/api/native/src/crawler.rs
@@ -581,9 +581,13 @@ fn _filter_url(data: FilterUrlCall) -> std::result::Result<FilterUrlResult, Stri
}
};
- if is_internal_link(&context_url, &base_url)
- && data.allow_external_content_links
+ // Allow an external destination when external content links are enabled and
+ // it is not an external site's homepage. Two cases qualify: the link was
+ // found on an in-scope page, or an already-admitted external link redirected
+ // within its own domain (i.e. resolving to its canonical URL).
+ if data.allow_external_content_links
&& !is_external_main_page(url_str)
+ && (is_internal_link(&context_url, &base_url) || is_internal_link(&url, &context_url))
{
return Ok(FilterUrlResult {
allowed: true,
@@ -1194,4 +1198,74 @@ mod tests {
assert!(!is_file("page"));
assert!(!is_file("directory/"));
}
+
+ fn filter_url_call(href: &str, url: &str, base_url: &str) -> FilterUrlCall {
+ FilterUrlCall {
+ href: href.to_string(),
+ url: url.to_string(),
+ base_url: base_url.to_string(),
+ excludes: vec![],
+ ignore_robots_txt: true,
+ robots_txt: "".to_string(),
+ robots_user_agent: None,
+ allow_external_content_links: true,
+ allow_subdomains: false,
+ }
+ }
+
+ // A discovered external link that redirects within its own domain (the
+ // redirect target's URL and its source URL are both external to the crawl
+ // seed) must be allowed under allowExternalLinks. Regression test for #4315.
+ #[test]
+ fn test_filter_url_allows_external_link_redirect_same_domain() {
+ let result = _filter_url(filter_url_call(
+ "http://www.iana.org/help/example-domains",
+ "https://iana.org/domains/example",
+ "https://example.org",
+ ))
+ .unwrap();
+ assert!(result.allowed);
+ assert!(result.denial_reason.is_none());
+ }
+
+ // Same as above but the redirect keeps the exact hostname, only changing the
+ // path (the london.gov.uk case from #4315).
+ #[test]
+ fn test_filter_url_allows_external_link_redirect_same_host_path_change() {
+ let result = _filter_url(filter_url_call(
+ "https://www.london.gov.uk/programmes-strategies/planning/london-plan",
+ "https://www.london.gov.uk/what-we-do/planning/london-plan",
+ "https://example.org",
+ ))
+ .unwrap();
+ assert!(result.allowed);
+ }
+
+ // An in-scope URL that redirects to an unrelated external homepage stays
+ // denied — the external-main-page exclusion is intentional (#4316).
+ #[test]
+ fn test_filter_url_denies_redirect_to_external_homepage() {
+ let result = _filter_url(filter_url_call(
+ "https://www.peoplefirstinfo.org.uk/",
+ "https://www.westminster.gov.uk/node/21229",
+ "https://www.westminster.gov.uk",
+ ))
+ .unwrap();
+ assert!(!result.allowed);
+ assert_eq!(result.denial_reason.unwrap(), "EXTERNAL_LINK");
+ }
+
+ // A redirect that hops to an unrelated external domain (not the source's own
+ // domain, not in scope) remains denied.
+ #[test]
+ fn test_filter_url_denies_redirect_to_unrelated_external_domain() {
+ let result = _filter_url(filter_url_call(
+ "https://unrelated.com/article",
+ "https://iana.org/domains/example",
+ "https://example.org",
+ ))
+ .unwrap();
+ assert!(!result.allowed);
+ assert_eq!(result.denial_reason.unwrap(), "EXTERNAL_LINK");
+ }
}
diff --git a/apps/api/src/__tests__/snips/v2/crawl.test.ts b/apps/api/src/__tests__/snips/v2/crawl.test.ts
index e6ebc0e2a..f41a21353 100644
--- a/apps/api/src/__tests__/snips/v2/crawl.test.ts
+++ b/apps/api/src/__tests__/snips/v2/crawl.test.ts
@@ -447,6 +447,41 @@ describe("Crawl tests", () => {
5 * scrapeTimeout,
);
+ // Regression for #4315: with allowExternalLinks, a discovered external link
+ // that redirects within its own domain must be followed and scraped, not
+ // rejected as EXTERNAL_LINK. example.org links to iana.org/domains/example,
+ // which redirects to www.iana.org/help/example-domains.
+ concurrentIf(!process.env.TEST_SUITE_SELF_HOSTED)(
+ "allowExternalLinks follows a redirecting external link",
+ async () => {
+ const res = await crawl(
+ {
+ url: "https://example.org/",
+ limit: 2,
+ maxDiscoveryDepth: 1,
+ allowExternalLinks: true,
+ sitemap: "skip",
+ },
+ identity,
+ );
+
+ expect(res.success).toBe(true);
+ if (res.success) {
+ const scrapedIana = res.data.some(page => {
+ try {
+ const host = new URL(page.metadata.url ?? page.metadata.sourceURL!)
+ .hostname;
+ return host === "iana.org" || host.endsWith(".iana.org");
+ } catch {
+ return false;
+ }
+ });
+ expect(scrapedIana).toBe(true);
+ }
+ },
+ 5 * scrapeTimeout,
+ );
+
describeIf(TEST_PRODUCTION || (HAS_AI && ALLOW_TEST_SUITE_WEBSITE))(
"Crawl API with Prompt",
() => {